Merge review R1 fixes: removed driver's resilience tracker released; SDK timed-unshelve no-op (engine owns expiry); commented re-ack stored as AddComment; native-ack Primary verdict published from start; drain gate SharesRowsWithPeer; malformed cache bytes don't adopt the revision; HostAbortedException passthrough; ClusterRoleInfo seed filtered

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-08 07:47:57 -04:00
parent b7a48e2b93 e3dcb180c3
commit b99aea46b2
28 changed files with 691 additions and 136 deletions
@@ -102,8 +102,14 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase

        // Barrier: diagnostics are served only after PreStart's boot work ran, so the absence below is
        // asserted after the boot path had its chance to Tell (positive control is the valid-bytes test).
-        AskDiagnostics(actor);
+        var snapshot = AskDiagnostics(actor);
        publishProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
+
+        // Review R1 m4: nothing was applied, so the node is NOT running from cache and has NOT adopted the
+        // cached revision (Stale with no configuration, retrying the ConfigDb) — the valid-bytes test above
+        // is the control that both flip when the bytes are usable.
+        snapshot.RunningFromCache.ShouldBeFalse();
+        snapshot.CurrentRevision.ShouldBeNull();
    }

    [Fact]
@@ -111,6 +111,39 @@ public sealed class DriverHostActorFetchAndCacheBootstrapTests : RuntimeActorTes
        fetcher.Calls.ShouldBe(1);
    }

+    /// <summary>
+    ///     #601 / review R1 m4 — a cache row whose bytes are malformed applied nothing, so its revision must not
+    ///     become current: the boot lands Steady-with-no-revision (as for an empty cache), and a re-dispatch of
+    ///     the SAME revision really applies (fetches) instead of being acked by the "already at this rev"
+    ///     short-circuit. <see cref="WithCachedArtifact_RestoresServedState_WithoutReadingCentralSql"/> is the
+    ///     control: valid bytes do adopt the revision.
+    /// </summary>
+    [Fact]
+    public void WithMalformedCachedBytes_TheRevisionIsNotAdopted_AndARedispatchOfItActuallyApplies()
+    {
+        var cached = new CachedDeploymentArtifact(
+            DeploymentId.NewId().ToString(), CachedRev.Value,
+            System.Text.Encoding.UTF8.GetBytes("{\"DriverInstances\": ["), DateTimeOffset.UtcNow.AddHours(-1));
+        var fetcher = new RecordingFetcher(Artifact());
+        var coordinator = CreateTestProbe();
+
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            new ThrowingDbFactory(), TestNode, coordinator.Ref,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: new StubArtifactCache(cached),
+            fetchAndCacheMode: true,
+            artifactFetcher: fetcher));
+
+        var snapshot = AskDiagnostics(actor);
+        snapshot.CurrentRevision.ShouldBeNull();
+        snapshot.RunningFromCache.ShouldBeFalse();
+
+        actor.Tell(new DispatchDeployment(DeploymentId.Parse(cached.DeploymentId), CachedRev, CorrelationId.NewId()));
+        coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
+        fetcher.Calls.ShouldBe(1);
+        AskDiagnostics(actor).CurrentRevision.ShouldBe(CachedRev);
+    }
+
    private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
    {
        var probe = CreateTestProbe();
@@ -0,0 +1,109 @@
+using Akka.Actor;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
+
+/// <summary>
+///     Review R1 m2 (Gitea #597) — the Primary data-plane verdict published to <see cref="IDriverPrimaryRoleSource"/>
+///     (read by the native-ack gate and the Sparkplug primary-host claim) mirrors the host's own live gate from the
+///     moment the host starts, not only from the first redundancy snapshot.
+/// </summary>
+/// <remarks>
+///     <c>RedundancyRoleView</c> is seeded with the single-node verdict (allow). Before this fix nothing published to
+///     it until a snapshot arrived, so on a pair both nodes accepted a native ack through the whole boot window while
+///     <c>DriverHostActor</c> — which evaluates the driver member count live — denied the upstream route on the
+///     non-Primary. Assertions are on what was PUBLISHED: the seed already reads <c>true</c>, so a current-value
+///     assertion of the single-node case would pass with no publish at all.
+/// </remarks>
+public sealed class DriverHostActorPrimaryRoleSourceTests : RuntimeActorTestBase
+{
+    private static readonly NodeId TestNode = NodeId.Parse("driver-primary-source-test");
+    private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
+
+    /// <summary>On a pair, an unknown role publishes DENY at start — before any snapshot. The later Primary snapshot
+    /// is the positive control: the source is wired, and the deny was the verdict, not a missing publish.</summary>
+    [Fact]
+    public void A_paired_host_publishes_deny_before_the_first_snapshot()
+    {
+        var source = new RecordingPrimaryRoleSource();
+        var host = SpawnHost(source, () => 2);
+
+        AwaitAssert(() => source.Published.ShouldBe([false]), duration: Timeout);
+
+        host.Tell(Snapshot(RedundancyRole.Primary));
+        AwaitAssert(() => source.Published.ShouldBe([false, true]), duration: Timeout);
+    }
+
+    /// <summary>A single-driver node keeps the allow posture — it is published, not merely seeded.</summary>
+    [Fact]
+    public void A_single_node_host_publishes_allow_at_start()
+    {
+        var source = new RecordingPrimaryRoleSource();
+        SpawnHost(source, () => 1);
+
+        AwaitAssert(() => source.Published.ShouldBe([true]), duration: Timeout);
+    }
+
+    /// <summary>A driver peer coming Up while the role is still unknown flips the published verdict to deny, with no
+    /// snapshot involved — the membership re-publish path.</summary>
+    [Fact]
+    public void A_peer_joining_before_any_snapshot_flips_the_published_verdict_to_deny()
+    {
+        var source = new RecordingPrimaryRoleSource();
+        var members = 1;
+        var host = SpawnHost(source, () => Volatile.Read(ref members));
+        AwaitAssert(() => source.Published.ShouldBe([true]), duration: Timeout);
+
+        Volatile.Write(ref members, 2);
+        host.Tell(DriverHostActor.RepublishPrimaryVerdict.Instance);
+
+        AwaitAssert(() => source.Published.ShouldBe([true, false]), duration: Timeout);
+    }
+
+    private IActorRef SpawnHost(IDriverPrimaryRoleSource source, Func<int> driverMemberCount) =>
+        Sys.ActorOf(DriverHostActor.Props(
+            NewInMemoryDbFactory(),
+            TestNode,
+            ackRouter: null,
+            driverMemberCountProvider: driverMemberCount,
+            primaryRoleSource: source));
+
+    private static RedundancyStateChanged Snapshot(RedundancyRole role) =>
+        new(
+            [
+                new NodeRedundancyState(TestNode, role,
+                    IsClusterLeader: role == RedundancyRole.Primary,
+                    IsDriverPrimary: role == RedundancyRole.Primary,
+                    AsOfUtc: DateTime.UtcNow),
+            ],
+            CorrelationId.NewId());
+
+    /// <summary>Remembers every published verdict, so tests assert on the actor's action, not on the seed.</summary>
+    private sealed class RecordingPrimaryRoleSource : IDriverPrimaryRoleSource
+    {
+        private readonly List<bool> _published = [];
+        private readonly Lock _gate = new();
+
+        /// <summary>Every value published so far, in order.</summary>
+        public IReadOnlyList<bool> Published
+        {
+            get { lock (_gate) return _published.ToArray(); }
+        }
+
+        public bool ShouldActAsPrimary
+        {
+            get { lock (_gate) return _published.Count == 0 || _published[^1]; }
+        }
+
+        public void PublishPrimaryRole(bool shouldActAsPrimary)
+        {
+            lock (_gate) _published.Add(shouldActAsPrimary);
+        }
+    }
+}
@@ -227,6 +227,77 @@ public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBas
        replayed.ShouldContain("other-node-driver");
    }

+    /// <summary>
+    ///     #609 (review R1 M1) — the Resilience half of the eviction must SURVIVE the next resilience-publisher
+    ///     tick. <c>DriverResilienceStatusPublisherService</c> re-emits <c>tracker.Snapshot()</c> into the hub
+    ///     every interval, so evicting only the hub cache was undone within one tick: the host must also
+    ///     release the instance's tracker state. The tick below is the publisher's own body
+    ///     (<c>BuildMessages</c> → <c>hub.Emit</c>) minus the DPS publish, run against the REAL factory +
+    ///     tracker the Host wires. The neighbour instance is the positive control: it is still emitted by
+    ///     the same tick, so an absent drv-1 is not an empty tick.
+    /// </summary>
+    [Fact]
+    public void Removing_a_driver_instance_releases_its_resilience_state_so_the_next_publisher_tick_omits_it()
+    {
+        var db = NewInMemoryDbFactory();
+        var factory = new SubscribingDriverFactory("Modbus");
+        var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
+        var tracker = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker();
+        var invokerFactory = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory(
+            new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResiliencePipelineBuilder(statusTracker: tracker),
+            tracker,
+            _ => ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTier.A);
+        var coordinator = CreateTestProbe();
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            db, TestNode, coordinator.Ref,
+            driverFactory: factory,
+            localRoles: new HashSet<string> { "driver" },
+            invokerFactory: invokerFactory,
+            telemetryHub: hub));
+
+        actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
+        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
+
+        // Resilience counters for the running driver and for an instance this host does not own.
+        tracker.RecordBreakerOpen("drv-1", "dev1", DateTime.UtcNow);
+        tracker.RecordFailure("neighbour-drv", "dev9", DateTime.UtcNow);
+        PublisherTick(tracker, hub);
+        ReplayedResilienceIds(hub).ShouldContain("drv-1"); // control: drv-1 is primed before the removal
+
+        actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
+        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
+        AwaitAssert(
+            () => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
+            duration: Timeout);
+
+        // The publisher's next tick, after the removal.
+        PublisherTick(tracker, hub);
+
+        var replayed = ReplayedResilienceIds(hub);
+        replayed.ShouldNotContain("drv-1");
+        replayed.ShouldContain("neighbour-drv");
+
+        static void PublisherTick(
+            ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker t,
+            ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub h)
+        {
+            var now = DateTime.UtcNow;
+            foreach (var e in t.Snapshot())
+                h.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Resilience(new(
+                    e.DriverInstanceId, e.HostName, e.Snapshot.IsBreakerOpen, e.Snapshot.ConsecutiveFailures,
+                    e.Snapshot.CurrentInFlight, e.Snapshot.LastBreakerOpenUtc, e.Snapshot.LastSampledUtc, now)));
+        }
+
+        static List<string> ReplayedResilienceIds(ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub h)
+        {
+            using var sub = h.Subscribe(64);
+            var ids = new List<string>();
+            while (sub.Reader.TryRead(out var i))
+                if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Resilience r) ids.Add(r.E.DriverInstanceId);
+            return ids;
+        }
+    }
+
    /// <summary>Spawns the host with the subscribing factory, dispatches <paramref name="deploymentId"/> and
    /// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.</summary>
    private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(