From 68588da05fa6af68fe38a62bdddb5b8544e8ef90 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 11:13:31 -0400 Subject: [PATCH] test(r2-04): 2-node delivery-path guard for the primary write gate (deferred-live) --- ...2-04-failure-visibility-plan.md.tasks.json | 2 +- .../PrimaryGateFailoverTests.cs | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/PrimaryGateFailoverTests.cs diff --git a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json index 3249880b..eb3fb3aa 100644 --- a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json +++ b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json @@ -14,7 +14,7 @@ { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "completed", "blockedBy": ["T9"] }, { "id": "T11", "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", "status": "completed", "blockedBy": ["T10"] }, { "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "completed", "blockedBy": ["T9"] }, - { "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "pending", "blockedBy": ["T11"] }, + { "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "deferred-live", "note": "rig/live; Host.IntegrationTests (~16GB/run leak + needs SQL/2-node rig) — implemented + build-verified, controller runs it in the serialized heavy pass", "blockedBy": ["T11"] }, { "id": "T14", "subject": "docs/Redundancy.md: gate policy table, boot-window client experience, new meters, Galaxy fail-closed semantics", "status": "pending", "blockedBy": [] }, { "id": "T15", "subject": "LIVE GATE: 2-node docker-dev rig verify (ServiceLevel 240/100, boot-window write rejected+reverted, single /alerts row)", "status": "pending", "blockedBy": ["T10", "T11", "T12", "T13"] } ] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/PrimaryGateFailoverTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/PrimaryGateFailoverTests.cs new file mode 100644 index 00000000..c9ca2b92 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/PrimaryGateFailoverTests.cs @@ -0,0 +1,75 @@ +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Runtime; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// archreview 03/S4 delivery-path guard — the "unit tests can't catch delivery" half. Proves that the +/// REAL DistributedPubSub-delivered redundancy snapshot (not a test-injected RedundancyStateChanged) +/// actually drives the Primary write gate end-to-end across a live 2-node Akka cluster: after both nodes +/// join and the snapshot converges, exactly ONE node's rejects a +/// with "not primary" (the Secondary) while the other +/// passes the gate (the Primary — its rejection, if any, is about node MAPPING, never the gate). +/// +/// Before the snapshot delivers, BOTH nodes have an unknown role on a 2-driver cluster and so both +/// default-DENY with "not primary (role unknown)" (archreview 03/S4). The poll below therefore +/// waits for the delivered snapshot to promote exactly one node to Primary — a broken redundancy-topic +/// subscribe (the historical double-break) would leave both stuck "role unknown" and time this out, which +/// is exactly the negative control this test provides over the pure-TestKit guards. +/// +[Trait("Category", "Failover")] +public sealed class PrimaryGateFailoverTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + // A NodeId that is guaranteed to have no driver mapping — so a node that PASSES the primary gate + // rejects it with a MAPPING reason ("no driver mapping for node …"), never a gate reason. + private const string UnmappedProbeNode = "__pgate_delivery_probe__"; + + [Fact] + public async Task Delivered_redundancy_snapshot_drives_the_primary_write_gate_across_the_cluster() + { + await using var harness = await TwoNodeClusterHarness.StartAsync(); + + var driverHostA = harness.NodeA.Services.GetRequiredService().Get(); + var driverHostB = harness.NodeB.Services.GetRequiredService().Get(); + + // Poll until the DELIVERED snapshot has promoted exactly one node to Primary: one host passes the + // gate (reason not "not primary*") and the other is gated ("not primary"). Deadline covers the + // ~250ms-debounced initial publish plus cluster convergence margin. + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(45); + (bool aGated, bool bGated, string? aReason, string? bReason) last = default; + while (DateTime.UtcNow < deadline) + { + var a = await RouteProbeWriteAsync(driverHostA); + var b = await RouteProbeWriteAsync(driverHostB); + last = (IsGateReject(a), IsGateReject(b), a.Reason, b.Reason); + + // Exactly one node gated ⇒ the snapshot converged (one Primary, one Secondary). + if (last.aGated ^ last.bGated) break; + await Task.Delay(500, Ct); + } + + // Exactly one node is the Secondary (gate-rejected); the other is the Primary (passed the gate — + // its reject, if any, is a MAPPING reason). This proves the delivered snapshot drives the gate. + (last.aGated ^ last.bGated).ShouldBeTrue( + $"exactly one node should be gated once the redundancy snapshot converges (A gated={last.aGated} reason='{last.aReason}', B gated={last.bGated} reason='{last.bReason}')"); + + var primaryReason = last.aGated ? last.bReason : last.aReason; + primaryReason.ShouldNotBeNull(); + primaryReason!.StartsWith("not primary", StringComparison.Ordinal).ShouldBeFalse( + $"the Primary must PASS the gate — its reject reason should be a mapping reason, not '{primaryReason}'"); + } + + private static async Task RouteProbeWriteAsync(IActorRef driverHost) + => await driverHost.Ask( + new DriverHostActor.RouteNodeWrite(UnmappedProbeNode, 0.0), TimeSpan.FromSeconds(10)); + + private static bool IsGateReject(DriverHostActor.NodeWriteResult r) + => !r.Success && r.Reason is not null && r.Reason.StartsWith("not primary", StringComparison.Ordinal); +}