Merge pull request 'Ensure driver test cleanup preserves primary failures' (#730) from test/task-assertion-return-discard into master

src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OpcUaApplicationHost.cs

@@ -69,8 +69,10 @@ public sealed class OpcUaApplicationHostOptions
     ///         Note this controls <i>admission</i>, not authority. An admitted anonymous session is a
     ///         normal principal (<c>anonymous</c>) whose grants come from <c>NodeAcl</c> rows naming it,
     ///         exactly like a key's principal — there is no anonymous bypass branch in the evaluator.
-    ///         Until per-node enforcement lands, an admitted session of ANY kind can still read the whole
+    ///         When a wired data-plane authorizer is enabled, per-node ACL checks apply to anonymous
-    ///         address space, because reads carry no authorization check yet.
+    ///         sessions, including reads. Reads denied by the node manager return <c>BadUserAccessDenied</c>.
+    ///         With the authorizer disabled or not wired, reads retain the legacy behaviour without
+    ///         per-node ACL enforcement.
     ///     </para>
     /// </remarks>
     public bool AllowAnonymous { get; set; } = true;

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs

@@ -147,6 +147,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     private readonly TimeSpan _staleWindow;
     private readonly TimeSpan _probeFreshnessWindow;
     private readonly TimeSpan _healthTickInterval;
+    private readonly Func<DateTime> _utcNow;
     private readonly Akka.Cluster.Cluster _cluster = Akka.Cluster.Cluster.Get(Context.System);
     private readonly ILoggingAdapter _log = Context.GetLogger();
 
@@ -200,9 +201,10 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     /// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
     /// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
     /// defaults to 30 seconds.</param>
-    /// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
+    /// <param name="healthTickInterval">The period between freshness ticks; defaults to 5 seconds.
-    /// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
+    /// DB-backed nodes Ask the probe for cached status; DB-less nodes recompute snapshot freshness.
-    /// started when <paramref name="dbHealthProbe"/> is null.</param>
+    /// Legacy nodes without a probe do not start a timer.</param>
+    /// <param name="utcNow">The UTC clock used for freshness; defaults to <see cref="DateTime.UtcNow"/>.</param>
     /// <param name="dbLess">Per-cluster mesh Phase 4: this is a driver-only node with no ConfigDb, so the
     /// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
     /// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
@@ -219,7 +221,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
         TimeSpan? probeFreshnessWindow = null,
         TimeSpan? healthTickInterval = null,
         bool dbLess = false,
-        IDataPlaneAuthoritySink? dataPlaneAuthority = null) =>
+        IDataPlaneAuthoritySink? dataPlaneAuthority = null,
+        Func<DateTime>? utcNow = null) =>
         Akka.Actor.Props.Create(() => new OpcUaPublishActor(
             sink ?? NullOpcUaAddressSpaceSink.Instance,
             serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -232,7 +235,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             probeFreshnessWindow,
             healthTickInterval,
             dbLess,
-            dataPlaneAuthority)).WithDispatcher(DispatcherId);
+            dataPlaneAuthority,
+            utcNow)).WithDispatcher(DispatcherId);
 
     /// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
     /// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
@@ -249,9 +253,10 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     /// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
     /// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
     /// defaults to 30 seconds.</param>
-    /// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
+    /// <param name="healthTickInterval">The period between freshness ticks; defaults to 5 seconds.
-    /// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
+    /// DB-backed nodes Ask the probe for cached status; DB-less nodes recompute snapshot freshness.
-    /// started when <paramref name="dbHealthProbe"/> is null.</param>
+    /// Legacy nodes without a probe do not start a timer.</param>
+    /// <param name="utcNow">The UTC clock used for freshness; defaults to <see cref="DateTime.UtcNow"/>.</param>
     /// <param name="dbLess">Per-cluster mesh Phase 4: this is a driver-only node with no ConfigDb, so the
     /// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
     /// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
@@ -269,7 +274,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
         TimeSpan? probeFreshnessWindow = null,
         TimeSpan? healthTickInterval = null,
         bool dbLess = false,
-        IDataPlaneAuthoritySink? dataPlaneAuthority = null) =>
+        IDataPlaneAuthoritySink? dataPlaneAuthority = null,
+        Func<DateTime>? utcNow = null) =>
         Akka.Actor.Props.Create(() => new OpcUaPublishActor(
             sink ?? NullOpcUaAddressSpaceSink.Instance,
             serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -282,7 +288,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             probeFreshnessWindow,
             healthTickInterval,
             dbLess,
-            dataPlaneAuthority));
+            dataPlaneAuthority,
+            utcNow));
 
     /// <summary>Initializes a new instance of the <see cref="OpcUaPublishActor"/> class.</summary>
     /// <param name="sink">The OPC UA address space sink.</param>
@@ -298,9 +305,10 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     /// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
     /// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
     /// defaults to 30 seconds.</param>
-    /// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
+    /// <param name="healthTickInterval">The period between freshness ticks; defaults to 5 seconds.
-    /// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
+    /// DB-backed nodes Ask the probe for cached status; DB-less nodes recompute snapshot freshness.
-    /// started when <paramref name="dbHealthProbe"/> is null.</param>
+    /// Legacy nodes without a probe do not start a timer.</param>
+    /// <param name="utcNow">The UTC clock used for freshness; defaults to <see cref="DateTime.UtcNow"/>.</param>
     /// <param name="dbLess">Per-cluster mesh Phase 4: this is a driver-only node with no ConfigDb, so the
     /// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
     /// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
@@ -316,7 +324,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
         TimeSpan? probeFreshnessWindow = null,
         TimeSpan? healthTickInterval = null,
         bool dbLess = false,
-        IDataPlaneAuthoritySink? dataPlaneAuthority = null)
+        IDataPlaneAuthoritySink? dataPlaneAuthority = null,
+        Func<DateTime>? utcNow = null)
     {
         _sink = sink;
         _dataPlaneAuthority = dataPlaneAuthority;
@@ -330,6 +339,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
         _staleWindow = staleWindow ?? TimeSpan.FromSeconds(30);
         _probeFreshnessWindow = probeFreshnessWindow ?? TimeSpan.FromSeconds(30);
         _healthTickInterval = healthTickInterval ?? TimeSpan.FromSeconds(5);
+        _utcNow = utcNow ?? (() => DateTime.UtcNow);
 
         Receive<AttributeValueUpdate>(HandleAttributeUpdate);
         Receive<AttributeValueBatchUpdate>(HandleAttributeBatchUpdate);
@@ -353,11 +363,10 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
         }
 
-        // Production delivery of DB-health: when a probe is wired, kick an immediate refresh + start the
+        // DB-backed nodes refresh their cached probe status; DB-less nodes still need periodic
-        // periodic tick so _lastDbHealth gets populated and kept fresh without any external pump. Gated
+        // snapshot/probe-verdict age checks even when no new messages arrive. Legacy nodes keep
-        // ONLY on the probe being present (independent of the DPS subscribe) so the calculator path works
+        // their role-only behavior without a timer.
-        // in tests too. No probe → stay on the legacy role-only seam (no timer).
+        if (_dbLess || _dbHealthProbe is not null)
-        if (_dbHealthProbe is not null)
         {
             Self.Tell(HealthTick.Instance);                                        // immediate first refresh
             Timers.StartPeriodicTimer("db-health", HealthTick.Instance, _healthTickInterval);
@@ -1006,7 +1015,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
         RecomputeServiceLevel();
     }
 
-    /// <summary>Periodic self-tick: Asks the local <see cref="DbHealthProbeActor"/> for its cached
+    /// <summary>Periodic self-tick: recomputes freshness directly when no DB probe exists; otherwise
+    /// Asks the local <see cref="DbHealthProbeActor"/> for its cached
     /// status and pipes the reply back to <see cref="Self"/>, where <see cref="HandleDbHealthStatus"/>
     /// caches it + recomputes. A hung/late probe (Ask timeout) pipes a <c>Reachable=false</c> status so
     /// the node FAIL-SAFE-DEMOTES rather than freezing the last-known-good. The continuation only
@@ -1015,11 +1025,17 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     /// touched here.</summary>
     private void OnHealthTick()
     {
-        _dbHealthProbe!.Ask<DbHealthProbeActor.DbHealthStatus>(
+        if (_dbHealthProbe is null)
+        {
+            RecomputeServiceLevel();
+            return;
+        }
+
+        _dbHealthProbe.Ask<DbHealthProbeActor.DbHealthStatus>(
                 DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1))
             .ContinueWith(t => t.IsCompletedSuccessfully
                 ? t.Result
-                : new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "db-health ask timeout"))
+                : new DbHealthProbeActor.DbHealthStatus(false, _utcNow(), "db-health ask timeout"))
             .PipeTo(Self);
     }
 
@@ -1032,7 +1048,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     {
         // The result targets the probed node. If it isn't me, it's about a peer — ignore it.
         if (_localNode is null || r.NodeId != _localNode.Value) return;
-        _probeAboutMe = (r.Ok, DateTime.UtcNow);
+        _probeAboutMe = (r.Ok, _utcNow());
         RecomputeServiceLevel();
     }
 
@@ -1044,7 +1060,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     private bool OpcUaProbeOk()
     {
         if (_probeAboutMe is not { } verdict) return true;
-        if (DateTime.UtcNow - verdict.At > _probeFreshnessWindow) return true;
+        if (_utcNow() - verdict.At > _probeFreshnessWindow) return true;
         return verdict.Ok;
     }
 
@@ -1081,7 +1097,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
         // wins, so a DB-less Detached node stays 0.
         if (_dbLess)
         {
-            var utcNow = DateTime.UtcNow;
+            var utcNow = _utcNow();
             var dbLessInputs = new NodeHealthInputs(
                 MemberState: SafeSelfStatus(),
                 DbReachable: true,
@@ -1101,7 +1117,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             return;
         }
 
-        var now = DateTime.UtcNow;
+        var now = _utcNow();
         var inputs = new NodeHealthInputs(
             MemberState: SafeSelfStatus(),
             DbReachable: _lastDbHealth.Reachable,

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/AbCipReinitializeReleasesStateTests.cs

@@ -1,4 +1,5 @@
 using Shouldly;
+using ZB.MOM.WW.OtOpcUa.Tests;
 using Xunit;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests;
@@ -8,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests;
 ///     orphaned the previous state's probe loop + handles. It now releases them first.
 /// </summary>
 [Trait("Category", "Unit")]
-public sealed class AbCipReinitializeReleasesStateTests
+public sealed class AbCipReinitializeReleasesStateTests(ITestOutputHelper output)
 {
     /// <summary>Initialize twice: the first state's probe loop has stopped and its probe handle is disposed.</summary>
     [Fact]
@@ -27,28 +28,94 @@ public sealed class AbCipReinitializeReleasesStateTests
                 ProbeTagPath = "@raw_cpu_type",
             },
         }, "drv-reinit", factory);
-        await drv.InitializeAsync("{}", CancellationToken.None);
+        await FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
 
-        var first = drv.GetDeviceState(host)!;
+            var first = drv.GetDeviceState(host)!;
-        // Positive control: the first probe loop is running and has initialised its runtime.
+            // Positive control: the first probe loop is running and has initialised its runtime.
+            var deadline = DateTime.UtcNow.AddSeconds(5);
+            while (!(factory.Tags.TryGetValue("@raw_cpu_type", out var t) && t.InitializeCount > 0))
+            {
+                if (DateTime.UtcNow > deadline) throw new TimeoutException("probe never started");
+                await Task.Delay(10);
+            }
+            var firstTag = factory.Tags["@raw_cpu_type"];
+            var firstLoop = first.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+            firstTag.Disposed.ShouldBeFalse();
+
+            await drv.InitializeAsync("{}", CancellationToken.None);
+
+            firstLoop.IsCompleted.ShouldBeTrue(); // joined
+            firstTag.Disposed.ShouldBeTrue();
+            drv.GetDeviceState(host).ShouldNotBeSameAs(first);
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"AbCip SecondInitialize cleanup failed: {failure}"));
+    }
+
+    /// <summary>Failure after a live first or replacement state must release both owned resources.</summary>
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Body_failure_releases_live_probe_resources(bool afterReplacement)
+    {
+        const string host = "ab://10.0.0.5/1,0";
+        var factory = new FakeAbCipTagFactory();
+        var drv = new AbCipDriver(new AbCipDriverOptions
+        {
+            Devices = [new AbCipDeviceOptions(host)],
+            Probe = new AbCipProbeOptions
+            {
+                Enabled = true,
+                Interval = TimeSpan.FromMilliseconds(20),
+                Timeout = TimeSpan.FromMilliseconds(200),
+                ProbeTagPath = "@raw_cpu_type",
+            },
+        }, "drv-reinit", factory);
+        Task? firstLoop = null, secondLoop = null;
+        FakeAbCipTag? firstTag = null, secondTag = null;
+        var sentinel = new InvalidOperationException("body sentinel");
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
+            await WaitForProbeAsync(() => factory.Tags.TryGetValue("@raw_cpu_type", out var tag) && tag.InitializeCount > 0);
+            firstTag = factory.Tags["@raw_cpu_type"];
+            firstLoop = drv.GetDeviceState(host)!.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+            firstTag.Disposed.ShouldBeFalse();
+            if (afterReplacement)
+            {
+                await drv.InitializeAsync("{}", CancellationToken.None);
+                await WaitForProbeAsync(() => factory.Tags.TryGetValue("@raw_cpu_type", out var tag) && !ReferenceEquals(tag, firstTag) && tag.InitializeCount > 0);
+                secondTag = factory.Tags["@raw_cpu_type"];
+                secondLoop = drv.GetDeviceState(host)!.ProbeTask;
+                _ = secondLoop.ShouldNotBeNull();
+                secondLoop!.IsCompleted.ShouldBeFalse();
+            }
+            throw sentinel;
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"AbCip failure fixture cleanup failed: {failure}")));
+        actual.ShouldBeSameAs(sentinel);
+        firstLoop!.IsCompleted.ShouldBeTrue();
+        firstTag!.Disposed.ShouldBeTrue();
+        if (afterReplacement)
+        {
+            secondLoop!.IsCompleted.ShouldBeTrue();
+            secondTag!.Disposed.ShouldBeTrue();
+        }
+    }
+
+    private static async Task WaitForProbeAsync(Func<bool> condition)
+    {
         var deadline = DateTime.UtcNow.AddSeconds(5);
-        while (!(factory.Tags.TryGetValue("@raw_cpu_type", out var t) && t.InitializeCount > 0))
+        while (!condition())
         {
             if (DateTime.UtcNow > deadline) throw new TimeoutException("probe never started");
             await Task.Delay(10);
         }
-        var firstTag = factory.Tags["@raw_cpu_type"];
-        var firstLoop = first.ProbeTask;
-        firstLoop.ShouldNotBeNull();
-        firstLoop!.IsCompleted.ShouldBeFalse();
-        firstTag.Disposed.ShouldBeFalse();
-
-        await drv.InitializeAsync("{}", CancellationToken.None);
-
-        firstLoop.IsCompleted.ShouldBeTrue(); // joined
-        firstTag.Disposed.ShouldBeTrue();
-        drv.GetDeviceState(host).ShouldNotBeSameAs(first);
-
-        await drv.ShutdownAsync(CancellationToken.None);
     }
+
 }

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/FixtureCleanupTests.cs

@@ -0,0 +1,92 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Tests;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests;
+
+[Trait("Category", "Unit")]
+public sealed class FixtureCleanupTests
+{
+    [Fact]
+    public async Task Successful_body_runs_one_awaited_cleanup()
+    {
+        var calls = 0;
+        var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var run = FixtureCleanup.RunAsync(() => Task.CompletedTask, async () =>
+        {
+            calls++;
+            await release.Task;
+        }, _ => throw new Exception("unexpected report"));
+
+        run.IsCompleted.ShouldBeFalse();
+        release.SetResult();
+        await run;
+        calls.ShouldBe(1);
+    }
+
+    [Fact]
+    public async Task Body_fault_remains_same_exception_and_original_throwing_stack()
+    {
+        var primary = new InvalidOperationException("body");
+        var cleanupCalls = 0;
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(
+            () => ThrowFromBody(primary),
+            () => { cleanupCalls++; return Task.CompletedTask; },
+            _ => throw new Exception("unexpected report")));
+
+        actual.ShouldBeSameAs(primary);
+        actual.StackTrace.ShouldNotBeNull().ShouldContain(nameof(ThrowFromBody));
+        cleanupCalls.ShouldBe(1);
+    }
+
+    [Fact]
+    public async Task Cleanup_only_fault_is_thrown_exactly()
+    {
+        var cleanup = new ApplicationException("cleanup");
+        var actual = await Should.ThrowAsync<ApplicationException>(() => FixtureCleanup.RunAsync(
+            () => Task.CompletedTask,
+            () => Task.FromException(cleanup),
+            _ => throw new Exception("unexpected report")));
+        actual.ShouldBeSameAs(cleanup);
+    }
+
+    [Fact]
+    public async Task Both_faults_preserve_primary_and_record_secondary_once()
+    {
+        var primary = new InvalidOperationException("body");
+        var cleanup = new ApplicationException("cleanup");
+        var reported = new List<Exception>();
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(
+            () => ThrowFromBody(primary),
+            () => Task.FromException(cleanup),
+            reported.Add));
+
+        actual.ShouldBeSameAs(primary);
+        actual.StackTrace.ShouldNotBeNull().ShouldContain(nameof(ThrowFromBody));
+        actual.Data[FixtureCleanup.SecondaryExceptionKey].ShouldBeSameAs(cleanup);
+        reported.Count.ShouldBe(1);
+        reported[0].ShouldBeSameAs(cleanup);
+    }
+
+    [Fact]
+    public async Task Reporting_fault_does_not_replace_primary()
+    {
+        var primary = new InvalidOperationException("body");
+        var cleanup = new ApplicationException("cleanup");
+        var reportCalls = 0;
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(
+            () => ThrowFromBody(primary),
+            () => Task.FromException(cleanup),
+            _ => { reportCalls++; throw new Exception("report"); }));
+
+        actual.ShouldBeSameAs(primary);
+        actual.Data[FixtureCleanup.SecondaryExceptionKey].ShouldBeSameAs(cleanup);
+        reportCalls.ShouldBe(1);
+    }
+
+    private static async Task ThrowFromBody(Exception failure)
+    {
+        await Task.Yield();
+        throw failure;
+    }
+}

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests.csproj

@@ -38,4 +38,7 @@
         <Compile Include="..\ZB.MOM.WW.OtOpcUa.Driver.Ab.Transport.Tests\Fakes\FakeCipTarget.cs" Link="Fakes\FakeCipTarget.cs" />
     </ItemGroup>
 
+    <ItemGroup>
+        <Compile Include="..\..\TestSupport\FixtureCleanup.cs" Link="TestSupport\FixtureCleanup.cs" />
+    </ItemGroup>
 </Project>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/AbLegacyReinitializeReleasesStateTests.cs

@@ -1,4 +1,5 @@
 using Shouldly;
+using ZB.MOM.WW.OtOpcUa.Tests;
 using Xunit;
 using ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.PlcFamilies;
 
@@ -10,7 +11,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests;
 ///     probe loop + runtime. It now releases them first.
 /// </summary>
 [Trait("Category", "Unit")]
-public sealed class AbLegacyReinitializeReleasesStateTests
+public sealed class AbLegacyReinitializeReleasesStateTests(ITestOutputHelper output)
 {
     /// <summary>Initialize twice: the first state's probe loop has stopped and its probe runtime is disposed.</summary>
     [Fact]
@@ -23,28 +24,88 @@ public sealed class AbLegacyReinitializeReleasesStateTests
             Devices = [new AbLegacyDeviceOptions(host, AbLegacyPlcFamily.Slc500)],
             Probe = new AbLegacyProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20), ProbeAddress = "S:0" },
         }, "drv-reinit", factory);
-        await drv.InitializeAsync("{}", CancellationToken.None);
+        await FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
 
-        var first = drv.GetDeviceState(host)!;
+            var first = drv.GetDeviceState(host)!;
-        // Positive control: the first state's probe loop is running and has created + initialised its runtime.
+            // Positive control: the first state's probe loop is running and has created + initialised its runtime.
+            var deadline = DateTime.UtcNow.AddSeconds(5);
+            while (!(factory.Tags.TryGetValue("S:0", out var t) && t.InitializeCount > 0))
+            {
+                if (DateTime.UtcNow > deadline) throw new TimeoutException("probe never started");
+                await Task.Delay(10);
+            }
+            var firstTag = factory.Tags["S:0"];
+            var firstLoop = first.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+            firstTag.Disposed.ShouldBeFalse();
+
+            await drv.InitializeAsync("{}", CancellationToken.None);
+
+            firstLoop.IsCompleted.ShouldBeTrue(); // joined
+            firstTag.Disposed.ShouldBeTrue();
+            drv.GetDeviceState(host).ShouldNotBeSameAs(first);
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"AbLegacy SecondInitialize cleanup failed: {failure}"));
+    }
+
+    /// <summary>Failure after a live first or replacement state must release both owned resources.</summary>
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Body_failure_releases_live_probe_resources(bool afterReplacement)
+    {
+        const string host = "ab://10.0.0.5/1,0";
+        var factory = new FakeAbLegacyTagFactory();
+        var drv = new AbLegacyDriver(new AbLegacyDriverOptions
+        {
+            Devices = [new AbLegacyDeviceOptions(host, AbLegacyPlcFamily.Slc500)],
+            Probe = new AbLegacyProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20), ProbeAddress = "S:0" },
+        }, "drv-reinit", factory);
+        Task? firstLoop = null, secondLoop = null;
+        FakeAbLegacyTag? firstTag = null, secondTag = null;
+        var sentinel = new InvalidOperationException("body sentinel");
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
+            await WaitForProbeAsync(() => factory.Tags.TryGetValue("S:0", out var tag) && tag.InitializeCount > 0);
+            firstTag = factory.Tags["S:0"];
+            firstLoop = drv.GetDeviceState(host)!.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+            firstTag.Disposed.ShouldBeFalse();
+            if (afterReplacement)
+            {
+                await drv.InitializeAsync("{}", CancellationToken.None);
+                await WaitForProbeAsync(() => factory.Tags.TryGetValue("S:0", out var tag) && !ReferenceEquals(tag, firstTag) && tag.InitializeCount > 0);
+                secondTag = factory.Tags["S:0"];
+                secondLoop = drv.GetDeviceState(host)!.ProbeTask;
+                _ = secondLoop.ShouldNotBeNull();
+                secondLoop!.IsCompleted.ShouldBeFalse();
+            }
+            throw sentinel;
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"AbLegacy failure fixture cleanup failed: {failure}")));
+        actual.ShouldBeSameAs(sentinel);
+        firstLoop!.IsCompleted.ShouldBeTrue();
+        firstTag!.Disposed.ShouldBeTrue();
+        if (afterReplacement)
+        {
+            secondLoop!.IsCompleted.ShouldBeTrue();
+            secondTag!.Disposed.ShouldBeTrue();
+        }
+    }
+
+    private static async Task WaitForProbeAsync(Func<bool> condition)
+    {
         var deadline = DateTime.UtcNow.AddSeconds(5);
-        while (!(factory.Tags.TryGetValue("S:0", out var t) && t.InitializeCount > 0))
+        while (!condition())
         {
             if (DateTime.UtcNow > deadline) throw new TimeoutException("probe never started");
             await Task.Delay(10);
         }
-        var firstTag = factory.Tags["S:0"];
-        var firstLoop = first.ProbeTask;
-        firstLoop.ShouldNotBeNull();
-        firstLoop!.IsCompleted.ShouldBeFalse();
-        firstTag.Disposed.ShouldBeFalse();
-
-        await drv.InitializeAsync("{}", CancellationToken.None);
-
-        firstLoop.IsCompleted.ShouldBeTrue(); // joined
-        firstTag.Disposed.ShouldBeTrue();
-        drv.GetDeviceState(host).ShouldNotBeSameAs(first);
-
-        await drv.ShutdownAsync(CancellationToken.None);
     }
+
 }

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests.csproj

@@ -38,4 +38,7 @@
         <None Include="..\ZB.MOM.WW.OtOpcUa.Driver.Ab.Transport.Tests\Captures\*.hex" Link="Captures\%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest"/>
     </ItemGroup>
 
+    <ItemGroup>
+        <Compile Include="..\..\TestSupport\FixtureCleanup.cs" Link="TestSupport\FixtureCleanup.cs" />
+    </ItemGroup>
 </Project>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReinitializeReleasesStateTests.cs

@@ -1,4 +1,5 @@
 using Shouldly;
+using ZB.MOM.WW.OtOpcUa.Tests;
 using Xunit;
 using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
 
@@ -10,7 +11,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
 ///     probe loop + client. It now releases them first.
 /// </summary>
 [Trait("Category", "Unit")]
-public sealed class FocasReinitializeReleasesStateTests
+public sealed class FocasReinitializeReleasesStateTests(ITestOutputHelper output)
 {
     /// <summary>Initialize twice: the first state's probe loop has finished and its client is disposed.</summary>
     [Fact]
@@ -25,25 +26,78 @@ public sealed class FocasReinitializeReleasesStateTests
             Probe = new FocasProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20), Timeout = TimeSpan.FromSeconds(2) },
         };
         var drv = new FocasDriver(options, "focas-reinit", factory);
-        await drv.InitializeAsync("{}", CancellationToken.None);
+        await FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
 
-        var first = drv.GetDeviceState(host)!;
+            var first = drv.GetDeviceState(host)!;
-        // Positive control: the first state's probe loop is demonstrably running and holds a live client.
+            // Positive control: the first state's probe loop is demonstrably running and holds a live client.
-        await WaitUntilAsync(() => factory.Clients.Count > 0 && factory.Clients[0].IsConnected);
+            await WaitUntilAsync(() => factory.Clients.Count > 0 && factory.Clients[0].IsConnected);
-        var firstLoop = first.ProbeTask;
+            var firstLoop = first.ProbeTask;
-        firstLoop.ShouldNotBeNull();
+            _ = firstLoop.ShouldNotBeNull();
-        firstLoop!.IsCompleted.ShouldBeFalse();
+            firstLoop!.IsCompleted.ShouldBeFalse();
-        var firstClient = factory.Clients[0];
+            var firstClient = factory.Clients[0];
 
-        await drv.InitializeAsync("{}", CancellationToken.None);
+            await drv.InitializeAsync("{}", CancellationToken.None);
 
-        firstLoop.IsCompleted.ShouldBeTrue(); // joined
+            firstLoop.IsCompleted.ShouldBeTrue(); // joined
-        firstClient.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
+            firstClient.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
+            firstClient.IsConnected.ShouldBeFalse();
+            drv.GetDeviceState(host).ShouldNotBeSameAs(first);
+            _ = drv.GetDeviceState(host)!.ProbeTask.ShouldNotBeNull(); // the new state is running
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"FOCAS SecondInitialize cleanup failed: {failure}"));
+    }
+
+
+    /// <summary>Failure after a live first or replacement state must release both owned resources.</summary>
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Body_failure_releases_live_probe_resources(bool afterReplacement)
+    {
+        const string host = "focas://10.0.0.5:8193";
+        var factory = new FakeFocasClientFactory();
+        var options = new FocasDriverOptions
+        {
+            Devices = [new FocasDeviceOptions(host)],
+            RawTags = [FocasRawTags.Entry(new FocasTagDefinition("R", host, "R100", FocasDataType.Byte))],
+            Probe = new FocasProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20), Timeout = TimeSpan.FromSeconds(2) },
+        };
+        var drv = new FocasDriver(options, "focas-reinit", factory);
+        Task? firstLoop = null, secondLoop = null;
+        FakeFocasClient? firstClient = null, secondClient = null;
+        var sentinel = new InvalidOperationException("body sentinel");
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
+            await WaitUntilAsync(() => factory.Clients.Count > 0 && factory.Clients[0].IsConnected);
+            firstClient = factory.Clients[0];
+            firstLoop = drv.GetDeviceState(host)!.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+            if (afterReplacement)
+            {
+                await drv.InitializeAsync("{}", CancellationToken.None);
+                await WaitUntilAsync(() => factory.Clients.Count > 1 && factory.Clients[1].IsConnected);
+                secondClient = factory.Clients[1];
+                secondLoop = drv.GetDeviceState(host)!.ProbeTask;
+                _ = secondLoop.ShouldNotBeNull();
+                secondLoop!.IsCompleted.ShouldBeFalse();
+            }
+            throw sentinel;
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"FOCAS failure fixture cleanup failed: {failure}")));
+        actual.ShouldBeSameAs(sentinel);
+        firstLoop!.IsCompleted.ShouldBeTrue();
+        firstClient!.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
         firstClient.IsConnected.ShouldBeFalse();
-        drv.GetDeviceState(host).ShouldNotBeSameAs(first);
+        if (afterReplacement)
-        drv.GetDeviceState(host)!.ProbeTask.ShouldNotBeNull(); // the new state is running
+        {
-
+            secondLoop!.IsCompleted.ShouldBeTrue();
-        await drv.ShutdownAsync(CancellationToken.None);
+            secondClient!.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
+            secondClient.IsConnected.ShouldBeFalse();
+        }
     }
 
     private static async Task WaitUntilAsync(Func<bool> cond)

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests.csproj

@@ -34,4 +34,7 @@
         <ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.FOCAS\ZB.MOM.WW.OtOpcUa.Driver.FOCAS.csproj"/>
     </ItemGroup>
 
+    <ItemGroup>
+        <Compile Include="..\..\TestSupport\FixtureCleanup.cs" Link="TestSupport\FixtureCleanup.cs" />
+    </ItemGroup>
 </Project>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectDriverLifecycleTests.cs

@@ -1,4 +1,5 @@
 using Shouldly;
+using ZB.MOM.WW.OtOpcUa.Tests;
 using Xunit;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 
@@ -16,7 +17,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
 ///     keeps talking to the old Agent, and a cache flush that wedges browse. Each of those is a
 ///     defect the happy-path assertions alone would pass straight over.
 /// </remarks>
-public sealed class MTConnectDriverLifecycleTests
+public sealed class MTConnectDriverLifecycleTests(ITestOutputHelper output)
 {
     private const string AgentUri = "http://fixture-agent:5000";
 
@@ -598,18 +599,79 @@ public sealed class MTConnectDriverLifecycleTests
     // same "browse must not be left permanently empty" obligation.
 
     [Fact]
-    public void Dropping_the_probe_cache_keeps_the_observation_index()
+    public async Task Dropping_the_probe_cache_keeps_the_observation_index()
     {
-        var (driver, _) = InitializedDriverAsync().GetAwaiter().GetResult();
+        var (driver, _) = NewDriver();
+        await FixtureCleanup.RunAsync(async () =>
+        {
+            await driver.InitializeAsync("{}", CancellationToken.None);
 
-        driver.InvalidateProbeModel();
+            driver.InvalidateProbeModel();
 
-        driver.CachedProbeModel.ShouldBeNull();
+            driver.CachedProbeModel.ShouldBeNull();
 
-        // The index is required for correctness (it IS the read surface) — never flushable.
+            // The index is required for correctness (it IS the read surface) — never flushable.
-        driver.ObservationIndex.Count.ShouldBe(FixtureDataItemCount);
+            driver.ObservationIndex.Count.ShouldBe(FixtureDataItemCount);
-        driver.ObservationIndex.Get("dev1_pos").StatusCode.ShouldBe(0u);
+            driver.ObservationIndex.Get("dev1_pos").StatusCode.ShouldBe(0u);
-        driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+            driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+        }, () => driver.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"MTConnect cache drop cleanup failed: {failure}"));
+    }
+
+    [Fact]
+    public async Task Body_failure_after_probe_scheduler_enters_stops_probe_and_disposes_client()
+    {
+        var client = CannedAgentClient.FromFixtures();
+        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var driver = new MTConnectDriver(Opts(), "mt1", _ => client,
+            probeScheduler: async (_, ct) =>
+            {
+                entered.TrySetResult();
+                try { await released.Task.WaitAsync(ct); }
+                catch (OperationCanceledException) when (ct.IsCancellationRequested)
+                {
+                    cancelled.TrySetResult();
+                    throw;
+                }
+            });
+        Task? probe = null;
+        var sentinel = new InvalidOperationException("body sentinel");
+        try
+        {
+            var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(async () =>
+            {
+                await driver.InitializeAsync("{}", CancellationToken.None);
+                await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+                probe = driver.ProbeLoopTask;
+                _ = probe.ShouldNotBeNull();
+                probe!.IsCompleted.ShouldBeFalse();
+                client.IsDisposed.ShouldBeFalse();
+                throw sentinel;
+            }, () => driver.ShutdownAsync(CancellationToken.None),
+                failure => output.WriteLine($"MTConnect probe fixture cleanup failed: {failure}")));
+            actual.ShouldBeSameAs(sentinel);
+            await cancelled.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+            probe!.IsCompleted.ShouldBeTrue();
+            client.DisposeCount.ShouldBe(1);
+        }
+        finally { released.TrySetResult(); }
+    }
+
+    [Fact]
+    public async Task Initialization_failure_disposes_the_client_inside_fixture_boundary()
+    {
+        var client = CannedAgentClient.FromFixtures();
+        var failure = new HttpRequestException("current unavailable");
+        client.CurrentFailure = failure;
+        var driver = new MTConnectDriver(Opts(), "mt1", _ => client);
+        var actual = await Should.ThrowAsync<HttpRequestException>(() => FixtureCleanup.RunAsync(
+            () => driver.InitializeAsync("{}", CancellationToken.None),
+            () => driver.ShutdownAsync(CancellationToken.None),
+            secondary => output.WriteLine($"MTConnect init fixture cleanup failed: {secondary}")));
+        actual.ShouldBeSameAs(failure);
+        client.DisposeCount.ShouldBe(1);
     }
 
     [Fact]

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests.csproj

@@ -39,4 +39,7 @@
         </Content>
     </ItemGroup>
 
+    <ItemGroup>
+        <Compile Include="..\..\TestSupport\FixtureCleanup.cs" Link="TestSupport\FixtureCleanup.cs" />
+    </ItemGroup>
 </Project>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusLifecycleHygieneTests.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Reflection;
 using Shouldly;
+using ZB.MOM.WW.OtOpcUa.Tests;
 using Xunit;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Driver.Modbus;
@@ -14,7 +15,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests;
 ///     unit test alongside them per Driver.Modbus-012.
 /// </summary>
 [Trait("Category", "Unit")]
-public sealed class ModbusLifecycleHygieneTests
+public sealed class ModbusLifecycleHygieneTests(ITestOutputHelper output)
 {
     private sealed class FakeTransport : IModbusTransport
     {
@@ -528,7 +529,7 @@ public sealed class ModbusLifecycleHygieneTests
 
     /// <summary>Verifies that DecodeBitArray rejects an empty bitmap with InvalidDataException.</summary>
     [Fact]
-    public void DecodeBitArray_rejects_an_empty_bitmap_with_InvalidDataException()
+    public async Task DecodeBitArray_rejects_an_empty_bitmap_with_InvalidDataException()
     {
         var decode = typeof(ModbusDriver).GetMethod(
             "DecodeBitArray", BindingFlags.NonPublic | BindingFlags.Static)!;
@@ -542,11 +543,49 @@ public sealed class ModbusLifecycleHygieneTests
             RawTags = ModbusRawTags.Entries([new ModbusTagDefinition("Coil", ModbusRegion.Coils, 0, ModbusDataType.Bool)]),
         };
         var drv = new ModbusDriver(opts, "modbus-1", _ => fake);
-        drv.InitializeAsync("{}", CancellationToken.None).GetAwaiter().GetResult();
+        await FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
 
-        var r = drv.ReadAsync(["Coil"], CancellationToken.None).GetAwaiter().GetResult();
+            var r = await drv.ReadAsync(["Coil"], CancellationToken.None);
-        // The empty-bitmap guard surfaces via the BadCommunicationError catch-all.
+            // The empty-bitmap guard surfaces via the BadCommunicationError catch-all.
-        r[0].StatusCode.ShouldBe(0x80050000u);
+            r[0].StatusCode.ShouldBe(0x80050000u);
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"Modbus empty bitmap cleanup failed: {failure}"));
+    }
+
+    [Fact]
+    public async Task Body_failure_after_probe_request_cancels_it_and_disposes_transport()
+    {
+        var fake = new EmptyBitTransport();
+        var opts = new ModbusDriverOptions
+        {
+            Host = "fake",
+            RawTags = ModbusRawTags.Entries([new ModbusTagDefinition("Coil", ModbusRegion.Coils, 0, ModbusDataType.Bool)]),
+        };
+        var drv = new ModbusDriver(opts, "modbus-1", _ => fake);
+        var sentinel = new InvalidOperationException("body sentinel");
+        try
+        {
+            var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(async () =>
+            {
+                await drv.InitializeAsync("{}", CancellationToken.None);
+                var result = await drv.ReadAsync(["Coil"], CancellationToken.None);
+                result[0].StatusCode.ShouldBe(0x80050000u);
+                fake.CoilRequestCount.ShouldBe(1);
+                await fake.ProbeEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+                fake.ProbeToken.IsCancellationRequested.ShouldBeFalse();
+                fake.DisposeCount.ShouldBe(0);
+                throw sentinel;
+            }, () => drv.ShutdownAsync(CancellationToken.None),
+                failure => output.WriteLine($"Modbus probe fixture cleanup failed: {failure}")));
+            actual.ShouldBeSameAs(sentinel);
+            fake.ProbeToken.IsCancellationRequested.ShouldBeTrue();
+            await fake.ProbeCancelled.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+            await fake.ProbeRequestCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+            fake.DisposeCount.ShouldBe(1);
+        }
+        finally { fake.ReleaseProbe.TrySetResult(); }
     }
 
     /// <summary>
@@ -556,6 +595,14 @@ public sealed class ModbusLifecycleHygieneTests
     /// </summary>
     private sealed class EmptyBitTransport : IModbusTransport
     {
+        public TaskCompletionSource ProbeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        public TaskCompletionSource ProbeCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        public TaskCompletionSource ProbeRequestCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        public TaskCompletionSource ReleaseProbe { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        public CancellationToken ProbeToken { get; private set; }
+        public int DisposeCount { get; private set; }
+        public int CoilRequestCount { get; private set; }
+
         /// <summary>Returns a completed task without performing any connection.</summary>
         /// <param name="ct">The cancellation token for the operation.</param>
         public Task ConnectAsync(CancellationToken ct) => Task.CompletedTask;
@@ -563,10 +610,29 @@ public sealed class ModbusLifecycleHygieneTests
         /// <param name="unitId">The Modbus unit ID.</param>
         /// <param name="pdu">The protocol data unit being sent.</param>
         /// <param name="ct">The cancellation token for the operation.</param>
-        public Task<byte[]> SendAsync(byte unitId, byte[] pdu, CancellationToken ct)
+        public async Task<byte[]> SendAsync(byte unitId, byte[] pdu, CancellationToken ct)
-            => Task.FromResult(new byte[] { pdu[0], 0 });
+        {
-        /// <summary>Completes the disposal without doing any work.</summary>
+            if (pdu[0] == 0x03)
-        public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+            {
+                ProbeToken = ct;
+                ProbeEntered.TrySetResult();
+                try { await ReleaseProbe.Task.WaitAsync(ct); }
+                catch (OperationCanceledException) when (ct.IsCancellationRequested)
+                {
+                    ProbeCancelled.TrySetResult();
+                    throw;
+                }
+                finally { ProbeRequestCompleted.TrySetResult(); }
+            }
+            if (pdu[0] == 0x01) CoilRequestCount++;
+            return new byte[] { pdu[0], 0 };
+        }
+        /// <summary>Records transport disposal.</summary>
+        public ValueTask DisposeAsync()
+        {
+            DisposeCount++;
+            return ValueTask.CompletedTask;
+        }
     }
 
     // -------------------- Finding -003 (volatile _health) --------------------

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests.csproj

@@ -29,4 +29,7 @@
         <ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Modbus\ZB.MOM.WW.OtOpcUa.Driver.Modbus.csproj"/>
     </ItemGroup>
 
+    <ItemGroup>
+        <Compile Include="..\..\TestSupport\FixtureCleanup.cs" Link="TestSupport\FixtureCleanup.cs" />
+    </ItemGroup>
 </Project>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReinitializeReleasesStateTests.cs

@@ -1,4 +1,5 @@
 using Shouldly;
+using ZB.MOM.WW.OtOpcUa.Tests;
 using Xunit;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
@@ -8,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
 ///     orphaned the previous state's probe loop + client. It now releases them first.
 /// </summary>
 [Trait("Category", "Unit")]
-public sealed class TwinCATReinitializeReleasesStateTests
+public sealed class TwinCATReinitializeReleasesStateTests(ITestOutputHelper output)
 {
     /// <summary>Initialize twice: the first state's probe loop has finished and its client is disposed.</summary>
     [Fact]
@@ -21,28 +22,88 @@ public sealed class TwinCATReinitializeReleasesStateTests
             Devices = [new TwinCATDeviceOptions(host)],
             Probe = new TwinCATProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20), Timeout = TimeSpan.FromSeconds(1) },
         }, "drv-reinit", factory);
-        await drv.InitializeAsync("{}", CancellationToken.None);
+        await FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
 
-        var first = drv.GetDeviceState(host)!;
+            var first = drv.GetDeviceState(host)!;
-        // Positive control: the first probe loop is running and holds a connected client.
+            // Positive control: the first probe loop is running and holds a connected client.
+            var deadline = DateTime.UtcNow.AddSeconds(5);
+            while (!(factory.Clients.Count > 0 && factory.Clients[0].IsConnected))
+            {
+                if (DateTime.UtcNow > deadline) throw new TimeoutException("probe never connected");
+                await Task.Delay(10);
+            }
+            var firstClient = factory.Clients[0];
+            var firstLoop = first.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+
+            await drv.InitializeAsync("{}", CancellationToken.None);
+
+            firstLoop.IsCompleted.ShouldBeTrue(); // joined
+            firstClient.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
+            firstClient.IsConnected.ShouldBeFalse();
+            drv.GetDeviceState(host).ShouldNotBeSameAs(first);
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"TwinCAT SecondInitialize cleanup failed: {failure}"));
+    }
+
+    /// <summary>Failure after a live first or replacement state must release both owned resources.</summary>
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Body_failure_releases_live_probe_resources(bool afterReplacement)
+    {
+        const string host = "ads://5.23.91.23.1.1:851";
+        var factory = new FakeTwinCATClientFactory();
+        var drv = new TwinCATDriver(new TwinCATDriverOptions
+        {
+            Devices = [new TwinCATDeviceOptions(host)],
+            Probe = new TwinCATProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20), Timeout = TimeSpan.FromSeconds(1) },
+        }, "drv-reinit", factory);
+        Task? firstLoop = null, secondLoop = null;
+        FakeTwinCATClient? firstClient = null, secondClient = null;
+        var sentinel = new InvalidOperationException("body sentinel");
+        var actual = await Should.ThrowAsync<InvalidOperationException>(() => FixtureCleanup.RunAsync(async () =>
+        {
+            await drv.InitializeAsync("{}", CancellationToken.None);
+            await WaitUntilAsync(() => factory.Clients.Count > 0 && factory.Clients[0].IsConnected);
+            firstClient = factory.Clients[0];
+            firstLoop = drv.GetDeviceState(host)!.ProbeTask;
+            _ = firstLoop.ShouldNotBeNull();
+            firstLoop!.IsCompleted.ShouldBeFalse();
+            if (afterReplacement)
+            {
+                await drv.InitializeAsync("{}", CancellationToken.None);
+                await WaitUntilAsync(() => factory.Clients.Count > 1 && factory.Clients[1].IsConnected);
+                secondClient = factory.Clients[1];
+                secondLoop = drv.GetDeviceState(host)!.ProbeTask;
+                _ = secondLoop.ShouldNotBeNull();
+                secondLoop!.IsCompleted.ShouldBeFalse();
+            }
+            throw sentinel;
+        }, () => drv.ShutdownAsync(CancellationToken.None),
+            failure => output.WriteLine($"TwinCAT failure fixture cleanup failed: {failure}")));
+        actual.ShouldBeSameAs(sentinel);
+        firstLoop!.IsCompleted.ShouldBeTrue();
+        firstClient!.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
+        firstClient.IsConnected.ShouldBeFalse();
+        if (afterReplacement)
+        {
+            secondLoop!.IsCompleted.ShouldBeTrue();
+            secondClient!.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
+            secondClient.IsConnected.ShouldBeFalse();
+        }
+    }
+
+    private static async Task WaitUntilAsync(Func<bool> condition)
+    {
         var deadline = DateTime.UtcNow.AddSeconds(5);
-        while (!(factory.Clients.Count > 0 && factory.Clients[0].IsConnected))
+        while (!condition())
         {
             if (DateTime.UtcNow > deadline) throw new TimeoutException("probe never connected");
             await Task.Delay(10);
         }
-        var firstClient = factory.Clients[0];
-        var firstLoop = first.ProbeTask;
-        firstLoop.ShouldNotBeNull();
-        firstLoop!.IsCompleted.ShouldBeFalse();
-
-        await drv.InitializeAsync("{}", CancellationToken.None);
-
-        firstLoop.IsCompleted.ShouldBeTrue(); // joined
-        firstClient.DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
-        firstClient.IsConnected.ShouldBeFalse();
-        drv.GetDeviceState(host).ShouldNotBeSameAs(first);
-
-        await drv.ShutdownAsync(CancellationToken.None);
     }
 }

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests.csproj

@@ -29,4 +29,7 @@
         <ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.TwinCAT\ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.csproj"/>
     </ItemGroup>
 
+    <ItemGroup>
+        <Compile Include="..\..\TestSupport\FixtureCleanup.cs" Link="TestSupport\FixtureCleanup.cs" />
+    </ItemGroup>
 </Project>

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorFreshnessTests.cs

@@ -0,0 +1,240 @@
+using System.Collections.Concurrent;
+using Akka.Actor;
+using Akka.TestKit;
+using Akka.TestKit.Xunit2;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer;
+using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Runtime.Health;
+using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
+
+public sealed class OpcUaPublishActorFreshnessTests : RuntimeActorTestBase
+{
+    [Theory]
+    [InlineData(false, 240, 200)]
+    [InlineData(true, 250, 210)]
+    public void DbLess_timer_demotes_without_incoming_messages(bool primary, byte fresh, byte stale)
+    {
+        var publisher = new RecordingPublisher();
+        var clock = new ControlledClock();
+        var local = NodeId.Parse("freshness-node");
+        var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: local, dbLess: true,
+            staleWindow: TimeSpan.FromSeconds(1), healthTickInterval: TimeSpan.FromMilliseconds(25),
+            utcNow: clock.Read));
+        actor.Tell(Snapshot(local, primary, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new[] { fresh }), PresenceBudget);
+        // Equality remains fresh. Observe several real ticks with a fixed clock; no actor Tell.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        var reads = clock.Reads;
+        AwaitCondition(() => clock.Reads >= reads + 4, PresenceBudget);
+        // Drain earlier self-enqueued publications without triggering a health recomputation.
+        actor.Tell(new Identify("freshness-barrier"), TestActor);
+        ExpectMsg<ActorIdentity>().Subject.ShouldBe(actor);
+        publisher.Levels.ShouldBe(new[] { fresh });
+        clock.Advance(TimeSpan.FromTicks(1));
+        // No further Tell: only the production timer can notice age crossing the window.
+        AwaitAssert(() => publisher.Levels.ShouldBe(new[] { fresh, stale }), PresenceBudget);
+        // A new fresh snapshot restores service, after which ticks continue to age that snapshot.
+        actor.Tell(Snapshot(local, primary, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new[] { fresh, stale, fresh }), PresenceBudget);
+        clock.Advance(TimeSpan.FromSeconds(1) + TimeSpan.FromTicks(1));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new[] { fresh, stale, fresh, stale }), PresenceBudget);
+    }
+
+    [Fact]
+    public void DbLess_timer_expires_negative_peer_verdict_only_after_exact_boundary()
+    {
+        var publisher = new RecordingPublisher();
+        var clock = new ControlledClock();
+        var local = NodeId.Parse("peer-verdict-node");
+        var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: local, dbLess: true,
+            staleWindow: TimeSpan.FromSeconds(10), probeFreshnessWindow: TimeSpan.FromSeconds(2),
+            healthTickInterval: TimeSpan.FromMilliseconds(25), utcNow: clock.Read));
+        actor.Tell(Snapshot(local, false, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240 }), PresenceBudget);
+        actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240, 0 }), PresenceBudget);
+        clock.Advance(TimeSpan.FromSeconds(2));
+        var reads = clock.Reads;
+        AwaitCondition(() => clock.Reads >= reads + 4, PresenceBudget);
+        // Drain earlier self-enqueued publications without triggering a health recomputation.
+        actor.Tell(new Identify("freshness-barrier"), TestActor);
+        ExpectMsg<ActorIdentity>().Subject.ShouldBe(actor);
+        publisher.Levels.ShouldBe(new byte[] { 240, 0 });
+        clock.Advance(TimeSpan.FromTicks(1));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240, 0, 240 }), PresenceBudget);
+    }
+
+    [Fact]
+    public void Stopping_DbLess_actor_cancels_freshness_timer_and_publications()
+    {
+        var publisher = new RecordingPublisher();
+        var clock = new ControlledClock();
+        var local = NodeId.Parse("stopped-node");
+        var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: local, dbLess: true,
+            staleWindow: TimeSpan.FromSeconds(1), healthTickInterval: TimeSpan.FromMilliseconds(25),
+            utcNow: clock.Read));
+        actor.Tell(Snapshot(local, false, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240 }), PresenceBudget);
+        var reads = clock.Reads;
+        AwaitCondition(() => clock.Reads >= reads + 4, PresenceBudget);
+        Watch(actor);
+        Sys.Stop(actor);
+        ExpectTerminated(actor);
+        reads = clock.Reads;
+        clock.Advance(TimeSpan.FromSeconds(2));
+        ExpectNoMsg(TimeSpan.FromMilliseconds(150)); // Six timer periods after confirmed termination.
+        clock.Reads.ShouldBe(reads);
+        publisher.Levels.ShouldBe(new byte[] { 240 });
+    }
+
+    [Fact]
+    public void DbBacked_timer_refreshes_probe_status_periodically()
+    {
+        var publisher = new RecordingPublisher();
+        var clock = new ControlledClock();
+        var local = NodeId.Parse("db-backed-periodic-node");
+        var db = CreateTestProbe();
+        var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: local, dbHealthProbe: db.Ref,
+            healthTickInterval: TimeSpan.FromMilliseconds(100), utcNow: clock.Read));
+        actor.Tell(Snapshot(local, false, clock.Now));
+        db.ExpectMsg<DbHealthProbeActor.GetStatus>();
+        db.Reply(new DbHealthProbeActor.DbHealthStatus(true, clock.Now, null));
+        AwaitAssert(() => publisher.Levels.ShouldContain((byte)240), PresenceBudget);
+        // A second request comes from the periodic timer, not the immediate startup tick.
+        db.ExpectMsg<DbHealthProbeActor.GetStatus>();
+        db.Reply(new DbHealthProbeActor.DbHealthStatus(false, clock.Now, "unreachable"));
+        AwaitAssert(() => publisher.Levels.Last().ShouldBe((byte)100), PresenceBudget);
+    }
+
+    [Fact]
+    public void DbBacked_timer_ask_timeout_demotes_last_known_healthy_node()
+    {
+        var publisher = new RecordingPublisher();
+        var clock = new ControlledClock();
+        var local = NodeId.Parse("db-backed-timeout-node");
+        var db = CreateTestProbe();
+        var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: local, dbHealthProbe: db.Ref,
+            healthTickInterval: TimeSpan.FromMilliseconds(100), utcNow: clock.Read));
+        actor.Tell(Snapshot(local, true, clock.Now));
+        db.ExpectMsg<DbHealthProbeActor.GetStatus>();
+        db.Reply(new DbHealthProbeActor.DbHealthStatus(true, clock.Now, null));
+        AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), PresenceBudget);
+        // Leave subsequent real Ask requests unanswered: the timeout status must reach the actor.
+        db.ExpectMsg<DbHealthProbeActor.GetStatus>();
+        AwaitAssert(() => publisher.Levels.Last().ShouldBe((byte)110), PresenceBudget);
+    }
+
+    [Fact]
+    public void Missing_local_snapshot_entry_retries_zero_on_timer_and_recovers()
+    {
+        var publisher = new RetryZeroPublisher();
+        var clock = new ControlledClock();
+        var local = NodeId.Parse("missing-entry-node");
+        var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: local, dbLess: true,
+            healthTickInterval: TimeSpan.FromMilliseconds(25), utcNow: clock.Read));
+        actor.Tell(Snapshot(local, false, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240 }), PresenceBudget);
+
+        // Only this snapshot Tell requests the first zero. Fail it and the next attempt so
+        // subsequent zero attempts must come from the periodic timer, exercising the null-entry
+        // branch repeatedly rather than mistaking the Detached branch for missing-node coverage.
+        actor.Tell(Snapshot(NodeId.Parse("other-node"), false, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240, 0 }), PresenceBudget);
+        publisher.ZeroAttempts.ShouldBeGreaterThanOrEqualTo(3);
+
+        actor.Tell(Snapshot(local, false, clock.Now));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240, 0, 240 }), PresenceBudget);
+        clock.Advance(TimeSpan.FromSeconds(30) + TimeSpan.FromTicks(1));
+        AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 240, 0, 240, 200 }), PresenceBudget);
+    }
+
+    [Fact]
+    public void No_snapshot_startup_does_not_publish_across_virtual_timer_periods()
+    {
+        // This actor requires the Cluster extension, but the absent/missing-entry guards do
+        // not inspect membership. Do not self-join: the ordinary harness waits for Up in its
+        // constructor, which cannot progress with a frozen scheduler.
+        using var local = new TestKit(AkkaTestHocon + @"
+            akka.scheduler.implementation = ""Akka.TestKit.TestScheduler, Akka.TestKit""");
+        var scheduler = (TestScheduler)local.Sys.Scheduler;
+        var publisher = new RetryZeroPublisher();
+        var clock = new ControlledClock();
+        var actor = local.ActorOfAsTestActorRef<OpcUaPublishActor>(OpcUaPublishActor.PropsForTests(
+            serviceLevel: publisher, localNode: NodeId.Parse("no-snapshot-node"), dbLess: true,
+            healthTickInterval: TimeSpan.FromSeconds(1), utcNow: clock.Read));
+
+        // TestActorRef uses the calling-thread dispatcher: Advance completes delivery before
+        // these assertions, including the immediate startup tick and three periodic ticks.
+        for (var period = 0; period < 3; period++)
+            scheduler.Advance(TimeSpan.FromSeconds(1));
+        publisher.Levels.ShouldBeEmpty();
+        publisher.ZeroAttempts.ShouldBe(0);
+        clock.Reads.ShouldBe(0);
+
+        // Positive control: the same registered timer must subsequently retry a failed zero.
+        // No whole snapshot at startup means no publication; a received snapshot missing the
+        // local entry means zero. Keep these two policies explicitly distinct.
+        actor.Tell(Snapshot(NodeId.Parse("other-node"), false, clock.Now));
+        publisher.ZeroAttempts.ShouldBe(1);
+        scheduler.Advance(TimeSpan.FromSeconds(1));
+        publisher.ZeroAttempts.ShouldBe(2);
+        publisher.Levels.ShouldBeEmpty();
+        scheduler.Advance(TimeSpan.FromSeconds(1));
+        publisher.ZeroAttempts.ShouldBe(3);
+        publisher.Levels.ShouldBe(new byte[] { 0 });
+        clock.Reads.ShouldBe(0);
+    }
+
+    private sealed class RetryZeroPublisher : IServiceLevelPublisher
+    {
+        private readonly ConcurrentQueue<byte> _levels = new();
+        private int _zeroAttempts;
+        public byte[] Levels => _levels.ToArray();
+        public int ZeroAttempts => Volatile.Read(ref _zeroAttempts);
+        public void Publish(byte serviceLevel)
+        {
+            if (serviceLevel == 0 && Interlocked.Increment(ref _zeroAttempts) <= 2)
+                throw new InvalidOperationException("test zero-publication failure");
+            _levels.Enqueue(serviceLevel);
+        }
+    }
+
+    private sealed class ControlledClock
+    {
+        private long _ticks = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).Ticks;
+        private int _reads;
+        public DateTime Now => new(Interlocked.Read(ref _ticks), DateTimeKind.Utc);
+        public int Reads => Volatile.Read(ref _reads);
+        public DateTime Read()
+        {
+            Interlocked.Increment(ref _reads);
+            return Now;
+        }
+        public void Advance(TimeSpan age) => Interlocked.Add(ref _ticks, age.Ticks);
+    }
+
+    private static RedundancyStateChanged Snapshot(NodeId local, bool primary, DateTime asOf) =>
+        new(new[] { new NodeRedundancyState(local,
+            primary ? RedundancyRole.Primary : RedundancyRole.Secondary,
+            IsClusterLeader: primary, IsDriverPrimary: primary, asOf) }, CorrelationId.NewId());
+
+    private sealed class RecordingPublisher : IServiceLevelPublisher
+    {
+        private readonly ConcurrentQueue<byte> _levels = new();
+        public byte[] Levels => _levels.ToArray();
+        public void Publish(byte serviceLevel) => _levels.Enqueue(serviceLevel);
+    }
+}

tests/TestSupport/FixtureCleanup.cs

@@ -0,0 +1,32 @@
+namespace ZB.MOM.WW.OtOpcUa.Tests;
+
+internal static class FixtureCleanup
+{
+    internal const string SecondaryExceptionKey = "FixtureCleanup.SecondaryException";
+
+    internal static async Task RunAsync(Func<Task> body, Func<Task> cleanup, Action<Exception> report)
+    {
+        Exception? primary = null;
+        try
+        {
+            await body();
+        }
+        catch (Exception failure)
+        {
+            primary = failure;
+            throw;
+        }
+        finally
+        {
+            try
+            {
+                await cleanup();
+            }
+            catch (Exception secondary) when (primary is not null)
+            {
+                try { primary.Data[SecondaryExceptionKey] = secondary; } catch { }
+                try { report(secondary); } catch { }
+            }
+        }
+    }
+}