Phase 6.3 Stream B + Stream D (core) — ServiceLevelCalculator + RecoveryStateManager + ApplyLeaseRegistry

Lands the pure-logic heart of Phase 6.3. OPC UA node wiring (Stream C),
RedundancyCoordinator topology loader (Stream A), Admin UI + metrics (Stream E),
and client interop tests (Stream F) are follow-up work — tracked as
tasks #145-150.

New Server.Redundancy sub-namespace:

- ServiceLevelCalculator — pure 8-state matrix per decision #154. Inputs:
  role, selfHealthy, peerUa/HttpHealthy, applyInProgress, recoveryDwellMet,
  topologyValid, operatorMaintenance. Output: OPC UA Part 5 §6.3.34 Byte.
  Reserved bands (0=Maintenance, 1=NoData, 2=InvalidTopology) override
  everything; operational bands occupy 30..255.
  Key invariants:
    * Authoritative-Primary = 255, Authoritative-Backup = 100.
    * Isolated-Primary = 230 (retains authority with peer down).
    * Isolated-Backup = 80 (does NOT auto-promote — non-transparent model).
    * Primary-Mid-Apply = 200, Backup-Mid-Apply = 50; apply dominates
      peer-unreachable per Stream C.4 integration expectation.
    * Recovering-Primary = 180, Recovering-Backup = 30.
    * Standalone treats healthy as Authoritative-Primary (no peer concept).
- ServiceLevelBand enum — labels every numeric band for logs + Admin UI.
  Values match the calculator table exactly; compliance script asserts
  drift detection.
- RecoveryStateManager — holds Recovering band until (dwell ≥ 60s default)
  AND (one publish witness observed). Re-fault resets both gates so a
  flapping node doesn't shortcut through recovery twice.
- ApplyLeaseRegistry — keyed on (ConfigGenerationId, PublishRequestId) per
  decision #162. BeginApplyLease returns an IAsyncDisposable so every exit
  path (success, exception, cancellation, dispose-twice) closes the lease.
  ApplyMaxDuration watchdog (10 min default) via PruneStale tick forces
  close after a crashed publisher so ServiceLevel can't stick at mid-apply.

Tests (40 new, all pass):
- ServiceLevelCalculatorTests (27): reserved bands override; self-unhealthy
  → NoData; invalid topology demotes both nodes to 2; authoritative primary
  255; backup 100; isolated primary 230 retains authority; isolated backup
  80 does not promote; http-only unreachable triggers isolated; mid-apply
  primary 200; mid-apply backup 50; apply dominates peer-unreachable; recovering
  primary 180; recovering backup 30; standalone treats healthy as 255;
  classify round-trips every band including Unknown sentinel.
- RecoveryStateManagerTests (6): never-faulted auto-meets dwell; faulted-only
  returns true (semantics-doc test — coordinator short-circuits on
  selfHealthy=false); recovered without witness never meets; witness without
  dwell never meets; witness + dwell-elapsed meets; re-fault resets.
- ApplyLeaseRegistryTests (7): empty registry not-in-progress; begin+dispose
  closes; dispose on exception still closes; dispose twice safe; concurrent
  leases isolated; watchdog closes stale; watchdog leaves recent alone.

Full solution dotnet test: 1137 passing (Phase 6.2 shipped at 1097, Phase 6.3
B + D core = +40 = 1137). Pre-existing Client.CLI Subscribe flake unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

tests/ZB.MOM.WW.OtOpcUa.Server.Tests/ApplyLeaseRegistryTests.cs

@@ -0,0 +1,118 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Server.Redundancy;
+
+namespace ZB.MOM.WW.OtOpcUa.Server.Tests;
+
+[Trait("Category", "Unit")]
+public sealed class ApplyLeaseRegistryTests
+{
+    private static readonly DateTime T0 = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
+
+    private sealed class FakeTimeProvider : TimeProvider
+    {
+        public DateTime Utc { get; set; } = T0;
+        public override DateTimeOffset GetUtcNow() => new(Utc, TimeSpan.Zero);
+    }
+
+    [Fact]
+    public async Task EmptyRegistry_NotInProgress()
+    {
+        var reg = new ApplyLeaseRegistry();
+        reg.IsApplyInProgress.ShouldBeFalse();
+        await Task.Yield();
+    }
+
+    [Fact]
+    public async Task BeginAndDispose_ClosesLease()
+    {
+        var reg = new ApplyLeaseRegistry();
+
+        await using (reg.BeginApplyLease(1, Guid.NewGuid()))
+        {
+            reg.IsApplyInProgress.ShouldBeTrue();
+            reg.OpenLeaseCount.ShouldBe(1);
+        }
+
+        reg.IsApplyInProgress.ShouldBeFalse();
+    }
+
+    [Fact]
+    public async Task Dispose_OnException_StillCloses()
+    {
+        var reg = new ApplyLeaseRegistry();
+        var publishId = Guid.NewGuid();
+
+        await Should.ThrowAsync<InvalidOperationException>(async () =>
+        {
+            await using var lease = reg.BeginApplyLease(1, publishId);
+            throw new InvalidOperationException("publish failed");
+        });
+
+        reg.IsApplyInProgress.ShouldBeFalse("await-using semantics must close the lease on exception");
+    }
+
+    [Fact]
+    public async Task Dispose_TwiceIsSafe()
+    {
+        var reg = new ApplyLeaseRegistry();
+        var lease = reg.BeginApplyLease(1, Guid.NewGuid());
+
+        await lease.DisposeAsync();
+        await lease.DisposeAsync();
+
+        reg.IsApplyInProgress.ShouldBeFalse();
+    }
+
+    [Fact]
+    public async Task MultipleLeases_Concurrent_StayIsolated()
+    {
+        var reg = new ApplyLeaseRegistry();
+        var id1 = Guid.NewGuid();
+        var id2 = Guid.NewGuid();
+
+        await using var lease1 = reg.BeginApplyLease(1, id1);
+        await using var lease2 = reg.BeginApplyLease(2, id2);
+
+        reg.OpenLeaseCount.ShouldBe(2);
+        await lease1.DisposeAsync();
+        reg.IsApplyInProgress.ShouldBeTrue("lease2 still open");
+        await lease2.DisposeAsync();
+        reg.IsApplyInProgress.ShouldBeFalse();
+    }
+
+    [Fact]
+    public async Task Watchdog_ClosesStaleLeases()
+    {
+        var clock = new FakeTimeProvider();
+        var reg = new ApplyLeaseRegistry(applyMaxDuration: TimeSpan.FromMinutes(10), timeProvider: clock);
+
+        _ = reg.BeginApplyLease(1, Guid.NewGuid()); // intentional leak; not awaited / disposed
+
+        // Lease still young → no-op.
+        clock.Utc = T0.AddMinutes(5);
+        reg.PruneStale().ShouldBe(0);
+        reg.IsApplyInProgress.ShouldBeTrue();
+
+        // Past the watchdog horizon → force-close.
+        clock.Utc = T0.AddMinutes(11);
+        var closed = reg.PruneStale();
+
+        closed.ShouldBe(1);
+        reg.IsApplyInProgress.ShouldBeFalse("ServiceLevel can't stick at mid-apply after a crashed publisher");
+        await Task.Yield();
+    }
+
+    [Fact]
+    public async Task Watchdog_LeavesRecentLeaseAlone()
+    {
+        var clock = new FakeTimeProvider();
+        var reg = new ApplyLeaseRegistry(applyMaxDuration: TimeSpan.FromMinutes(10), timeProvider: clock);
+
+        await using var lease = reg.BeginApplyLease(1, Guid.NewGuid());
+        clock.Utc = T0.AddMinutes(3);
+
+        reg.PruneStale().ShouldBe(0);
+        reg.IsApplyInProgress.ShouldBeTrue();
+    }
+}