feat(resilience): route IDriver.InitializeAsync through the capability invoker (R2-10 T10 closed)

The connect path was the one capability call the resilience pipeline never saw, so a driver that could not connect contributed zero breaker input and the /hosts resilience panel read 'All targets healthy' against a dead endpoint. InitializeAsync now routes through the invoker as DriverCapability.Initialize, keyed on the driver-instance id. - No pipeline retry: the actor's RetryConnect timer is the retry loop (parser forces retryCount overrides to 0, same funnel as write-shaped). - Breaker threshold 2, not the Read-sized 5: the breaker samples 30 s and the reconnect timer fires every 10 s, so at most three attempts fit one window — threshold 5 could never open. Pinned by its own test. - An open breaker fast-fails into the same InitializeFailed path as a real connect failure; the 15 s break window costs at most one extra tick. The best-effort teardown ShutdownAsync stays unwrapped. - Companion fix: DriverCapabilityInvokerFactory.Create now invalidates the instance's tracker snapshots with its pipelines — a respawn during an outage discarded the only breaker whose OnClosed could clear the OPEN flag, stranding a false 'Breaker OPEN' chip forever. Live gate PASSED on docker-dev (image symbol-verified per container): rtu-gate against dead 10.100.0.35:5021 opened its Initialize breaker on both centrals with the red 'BREAKER OPEN' chip on /hosts; a socat 5021→5020 forwarder recovered it (both breakers closed within 1 s, panel back to 'All targets healthy'); healthy MAIN-modbus recorded zero breaker events. Full suite 50/50 green. Evidence: docs/plans/2026-08-07-adminui-followups-and-live-gates.md §7.
2026-08-08 05:45:08 -04:00
parent c9f182b2d7
commit dac7e1885d
18 changed files with 445 additions and 8 deletions
@@ -22,6 +22,29 @@ public sealed class DriverCapabilityInvokerFactoryTests
        ILogger? logger = null) =>
        new(builder, new DriverResilienceStatusTracker(), _ => tier, logger);

+    /// <summary>
+    ///     A respawn (every <c>Create</c>) resets the instance's tracker state alongside its pipeline
+    ///     cache. The pipelines carry the actual breaker state and are discarded by the same call, so a
+    ///     snapshot recorded against them — a "Breaker OPEN" from a pre-deploy outage — refers to a
+    ///     breaker that no longer exists; keeping it would show a false OPEN chip forever, because only
+    ///     the discarded pipeline's <c>OnClosed</c> could have cleared it.
+    /// </summary>
+    [Fact]
+    public void Create_resets_the_instances_tracker_state_with_its_pipelines()
+    {
+        var tracker = new DriverResilienceStatusTracker();
+        var factory = new DriverCapabilityInvokerFactory(
+            new DriverResiliencePipelineBuilder(), tracker, _ => DriverTier.A);
+        tracker.RecordBreakerOpen("d1", "d1", DateTime.UtcNow);
+        tracker.RecordBreakerOpen("d2", "d2", DateTime.UtcNow);
+
+        factory.Create("d1", "Modbus");
+
+        tracker.TryGet("d1", "d1").ShouldBeNull(
+            "the respawn discarded the pipeline whose breaker this snapshot described");
+        tracker.TryGet("d2", "d2")!.IsBreakerOpen.ShouldBeTrue("other instances must be untouched");
+    }
+
    /// <summary>
    ///     End-to-end proof the per-instance override REACHES execution: a ResilienceConfig that sets
    ///     Read's retryCount to 0 must suppress the tier-A Read retry — a failing Read is attempted once,
@@ -331,6 +331,19 @@ public sealed class DriverResilienceOptionsParserTests
        diag.ShouldContain("AlarmAcknowledge never retries");
    }

+    /// <summary>An Initialize retryCount override is forced to 0 with a diagnostic — the
+    /// DriverInstanceActor reconnect timer is the retry loop, so pipeline retries would stack.</summary>
+    [Fact]
+    public void InitializeRetryOverride_IsForcedToZero_WithDiagnostic()
+    {
+        var options = DriverResilienceOptionsParser.ParseOrDefaults(
+            DriverTier.A, "{\"capabilityPolicies\":{\"Initialize\":{\"retryCount\":3}}}", out var diag);
+
+        options.Resolve(DriverCapability.Initialize).RetryCount.ShouldBe(0);
+        diag.ShouldNotBeNull();
+        diag.ShouldContain("Initialize never retries");
+    }
+
    /// <summary>A Read (idempotent) retryCount override is still honored — the invariant is write-shaped only.</summary>
    [Fact]
    public void ReadRetryOverride_StillHonored()
@@ -78,6 +78,39 @@ public sealed class DriverResilienceOptionsTests
        defaults[DriverCapability.AlarmAcknowledge].RetryCount.ShouldBe(0);
    }

+    /// <summary>
+    ///     Initialize never retries by default — the <c>DriverInstanceActor</c> reconnect timer IS the
+    ///     retry loop; pipeline retries would stack inside each timer tick.
+    /// </summary>
+    /// <param name="tier">The driver tier to test.</param>
+    [Theory]
+    [InlineData(DriverTier.A)]
+    [InlineData(DriverTier.B)]
+    [InlineData(DriverTier.C)]
+    public void Initialize_NeverRetries_ByDefault(DriverTier tier)
+    {
+        var defaults = DriverResilienceOptions.GetTierDefaults(tier);
+        defaults[DriverCapability.Initialize].RetryCount.ShouldBe(0);
+    }
+
+    /// <summary>
+    ///     The Initialize breaker threshold must be reachable at the reconnect cadence (R2-10 T10):
+    ///     the breaker samples over 30 s and the reconnect timer fires every 10 s, so at most THREE
+    ///     init attempts fit one sampling window. A threshold above 3 can never open on connect
+    ///     failures — the exact silent-inertness this capability exists to fix — so the default is
+    ///     pinned at 2 (Polly's minimum enabled throughput). Tier C stays breaker-less, consistent
+    ///     with every other Tier C capability (the supervisor's process-level breaker owns it).
+    /// </summary>
+    [Theory]
+    [InlineData(DriverTier.A, 2)]
+    [InlineData(DriverTier.B, 2)]
+    [InlineData(DriverTier.C, 0)]
+    public void Initialize_BreakerThreshold_IsReachableAtTheReconnectCadence(DriverTier tier, int expectedThreshold)
+    {
+        var defaults = DriverResilienceOptions.GetTierDefaults(tier);
+        defaults[DriverCapability.Initialize].BreakerFailureThreshold.ShouldBe(expectedThreshold);
+    }
+
    /// <summary>Verifies that idempotent capabilities retry by default.</summary>
    /// <param name="tier">The driver tier to test.</param>
    /// <param name="capability">The driver capability to test.</param>
@@ -315,6 +315,47 @@ public sealed class DriverResiliencePipelineBuilderTests
        snap.IsBreakerOpen.ShouldBeFalse("the breaker must read closed after a successful recovery probe");
    }

+    /// <summary>
+    ///     R2-10 T10 closure: with the TIER DEFAULTS (no override), consecutive Initialize failures
+    ///     open the breaker — and a subsequent attempt fast-fails WITHOUT reaching the call site.
+    ///     The load-bearing half is the threshold: at the 10 s reconnect cadence at most three init
+    ///     attempts fit the 30 s sampling window, so this test drives exactly the tier-default
+    ///     threshold of failures and demands an open breaker — it goes red if anyone raises the
+    ///     Initialize threshold back into never-opens territory (5), the silent inertness the
+    ///     live gate proved.
+    /// </summary>
+    [Fact]
+    public async Task Initialize_BreakerOpens_AtTierDefaults_AndFastFailsTheNextAttempt()
+    {
+        var tracker = new DriverResilienceStatusTracker();
+        var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
+        var pipeline = builder.GetOrCreate("drv-init", "drv-init", DriverCapability.Initialize, TierAOptions);
+
+        var threshold = TierAOptions.Resolve(DriverCapability.Initialize).BreakerFailureThreshold;
+        for (var i = 0; i < threshold; i++)
+        {
+            await Should.ThrowAsync<InvalidOperationException>(async () =>
+                await pipeline.ExecuteAsync(async _ =>
+                {
+                    await Task.Yield();
+                    throw new InvalidOperationException("connect refused");
+                }));
+        }
+
+        tracker.TryGet("drv-init", "drv-init")!.IsBreakerOpen.ShouldBeTrue(
+            "the tier-default threshold of consecutive connect failures must open the Initialize breaker");
+
+        // The open breaker fast-fails the next attempt without invoking the connect at all.
+        var reached = false;
+        await Should.ThrowAsync<BrokenCircuitException>(async () =>
+            await pipeline.ExecuteAsync(_ =>
+            {
+                reached = true;
+                return ValueTask.CompletedTask;
+            }));
+        reached.ShouldBeFalse("an open breaker must not let the connect attempt through");
+    }
+
    /// <summary>Verifies that the tracker isolates counters per host.</summary>
    [Fact]
    public async Task Tracker_IsolatesCounters_PerHost()
@@ -90,6 +90,27 @@ public sealed class DriverResilienceStatusTrackerTests
        tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
    }

+    /// <summary>
+    ///     Invalidate drops every entry for one instance and leaves other instances alone. Without
+    ///     this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale
+    ///     "Breaker OPEN" snapshot: the respawn discards the pipelines whose <c>OnClosed</c> is the
+    ///     only thing that stamps the close, so nothing can ever clear the flag again.
+    /// </summary>
+    [Fact]
+    public void Invalidate_DropsTheInstance_LeavesOthers()
+    {
+        var tracker = new DriverResilienceStatusTracker();
+        tracker.RecordBreakerOpen("drv", "host-a", Now);
+        tracker.RecordBreakerOpen("drv", "host-b", Now);
+        tracker.RecordBreakerOpen("other", "host-a", Now);
+
+        tracker.Invalidate("drv");
+
+        tracker.TryGet("drv", "host-a").ShouldBeNull();
+        tracker.TryGet("drv", "host-b").ShouldBeNull();
+        tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue();
+    }
+
    /// <summary>Verifies that RecordRecycle populates the LastRecycleUtc timestamp.</summary>
    [Fact]
    public void RecordRecycle_Populates_LastRecycleUtc()
@@ -80,6 +80,100 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
        invoker.WriteCalls.ShouldContain(c => c.Host == "plc-7" && c.IsIdempotent == false);
    }

+    /// <summary>
+    ///     R2-10 T10 closure: <c>IDriver.InitializeAsync</c> routes through the invoker as
+    ///     <see cref="DriverCapability.Initialize"/> keyed on the driver-instance id (init spans the
+    ///     whole driver, like a bulk subscribe). Before this wiring the connect path was the ONE
+    ///     capability call the invoker never saw — a driver that could not connect contributed zero
+    ///     breaker input, so the resilience panel stayed quiet for a dead endpoint.
+    /// </summary>
+    [Fact]
+    public void Initialize_routes_through_the_capability_invoker()
+    {
+        var invoker = new RecordingInvoker();
+        var driver = new HostResolvingWritableDriver();
+        var actor = Sys.ActorOf(DriverInstanceActor.Props(driver, invoker: invoker));
+
+        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
+
+        AwaitCondition(
+            () => invoker.Calls.Any(c => c.Capability == DriverCapability.Initialize),
+            TimeSpan.FromSeconds(3));
+        invoker.Calls.ShouldContain(c =>
+            c.Capability == DriverCapability.Initialize && c.Host == driver.DriverInstanceId);
+    }
+
+    /// <summary>
+    ///     The open-breaker shape end to end at the actor: when the INVOKER itself throws on the
+    ///     Initialize route (exactly what an open circuit does — the call site is never reached),
+    ///     the actor treats it as an ordinary init failure — lands in Reconnecting, keeps the retry
+    ///     timer running, and recovers to Connected once the invoker lets an attempt through. This
+    ///     pins two things: the reconnect-RETRY path is wrapped too (not just the initial connect),
+    ///     and a breaker fast-fail can never wedge or crash the actor.
+    /// </summary>
+    [Fact]
+    public async Task An_invoker_fast_fail_on_init_keeps_the_actor_retrying_until_recovery()
+    {
+        var invoker = new FastFailingInitInvoker(failFirst: 2);
+        var driver = new HostResolvingWritableDriver();
+        var actor = Sys.ActorOf(DriverInstanceActor.Props(
+            driver, invoker: invoker, reconnectInterval: TimeSpan.FromMilliseconds(100)));
+
+        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
+
+        // The first two init attempts die inside the invoker (open breaker); recovery is proven by a
+        // write succeeding — WriteAttribute fast-fails synchronously in Connecting/Reconnecting, so a
+        // Success reply is only possible from Connected.
+        DriverInstanceActor.WriteAttributeResult? reply = null;
+        for (var i = 0; i < 120 && reply is not { Success: true }; i++)
+        {
+            reply = await actor.Ask<DriverInstanceActor.WriteAttributeResult>(
+                new DriverInstanceActor.WriteAttribute("tag-1", 42), TimeSpan.FromSeconds(1));
+            if (reply is not { Success: true }) await Task.Delay(25);
+        }
+
+        reply.ShouldNotBeNull();
+        reply!.Success.ShouldBeTrue("the actor must recover to Connected once the breaker lets an attempt through");
+        invoker.InitializeAttempts.ShouldBeGreaterThanOrEqualTo(3,
+            "the reconnect-retry path must route through the invoker (initial attempt + at least the two fast-failed retries)");
+    }
+
+    /// <summary>Stands in for an OPEN circuit breaker on the Initialize route: the first
+    /// <paramref name="failFirst"/> Initialize calls throw WITHOUT invoking the call site (the
+    /// driver's connect never runs — exactly Polly's <c>BrokenCircuitException</c> shape), then the
+    /// breaker "closes" and calls pass through. All other capabilities always pass through.</summary>
+    private sealed class FastFailingInitInvoker : IDriverCapabilityInvoker
+    {
+        private readonly int _failFirst;
+        private int _initializeAttempts;
+
+        public FastFailingInitInvoker(int failFirst) => _failFirst = failFirst;
+
+        public int InitializeAttempts => Volatile.Read(ref _initializeAttempts);
+
+        public ValueTask<TResult> ExecuteAsync<TResult>(
+            DriverCapability capability, string hostName,
+            Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
+            => callSite(cancellationToken);
+
+        public ValueTask ExecuteAsync(
+            DriverCapability capability, string hostName,
+            Func<CancellationToken, ValueTask> callSite, CancellationToken cancellationToken)
+        {
+            if (capability == DriverCapability.Initialize
+                && Interlocked.Increment(ref _initializeAttempts) <= _failFirst)
+            {
+                throw new InvalidOperationException("circuit open (simulated fast-fail)");
+            }
+            return callSite(cancellationToken);
+        }
+
+        public ValueTask<TResult> ExecuteWriteAsync<TResult>(
+            string hostName, bool isIdempotent,
+            Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
+            => callSite(cancellationToken);
+    }
+
    /// <summary>Records every capability + resolved-host key routed through it, then delegates to the
    /// real call-site so the driver still runs. Stands in for <c>CapabilityInvoker</c> without pulling
    /// Polly / Core into the Runtime test.</summary>