feat(resilience): route IDriver.InitializeAsync through the capability invoker (R2-10 T10 closed)

The connect path was the one capability call the resilience pipeline never saw, so a driver that could not connect contributed zero breaker input and the /hosts resilience panel read 'All targets healthy' against a dead endpoint. InitializeAsync now routes through the invoker as DriverCapability.Initialize, keyed on the driver-instance id. - No pipeline retry: the actor's RetryConnect timer is the retry loop (parser forces retryCount overrides to 0, same funnel as write-shaped). - Breaker threshold 2, not the Read-sized 5: the breaker samples 30 s and the reconnect timer fires every 10 s, so at most three attempts fit one window — threshold 5 could never open. Pinned by its own test. - An open breaker fast-fails into the same InitializeFailed path as a real connect failure; the 15 s break window costs at most one extra tick. The best-effort teardown ShutdownAsync stays unwrapped. - Companion fix: DriverCapabilityInvokerFactory.Create now invalidates the instance's tracker snapshots with its pipelines — a respawn during an outage discarded the only breaker whose OnClosed could clear the OPEN flag, stranding a false 'Breaker OPEN' chip forever. Live gate PASSED on docker-dev (image symbol-verified per container): rtu-gate against dead 10.100.0.35:5021 opened its Initialize breaker on both centrals with the red 'BREAKER OPEN' chip on /hosts; a socat 5021→5020 forwarder recovered it (both breakers closed within 1 s, panel back to 'All targets healthy'); healthy MAIN-modbus recorded zero breaker events. Full suite 50/50 green. Evidence: docs/plans/2026-08-07-adminui-followups-and-live-gates.md §7.
2026-08-08 05:45:08 -04:00
parent c9f182b2d7
commit dac7e1885d
18 changed files with 445 additions and 8 deletions
@@ -22,6 +22,29 @@ public sealed class DriverCapabilityInvokerFactoryTests
        ILogger? logger = null) =>
        new(builder, new DriverResilienceStatusTracker(), _ => tier, logger);

+    /// <summary>
+    ///     A respawn (every <c>Create</c>) resets the instance's tracker state alongside its pipeline
+    ///     cache. The pipelines carry the actual breaker state and are discarded by the same call, so a
+    ///     snapshot recorded against them — a "Breaker OPEN" from a pre-deploy outage — refers to a
+    ///     breaker that no longer exists; keeping it would show a false OPEN chip forever, because only
+    ///     the discarded pipeline's <c>OnClosed</c> could have cleared it.
+    /// </summary>
+    [Fact]
+    public void Create_resets_the_instances_tracker_state_with_its_pipelines()
+    {
+        var tracker = new DriverResilienceStatusTracker();
+        var factory = new DriverCapabilityInvokerFactory(
+            new DriverResiliencePipelineBuilder(), tracker, _ => DriverTier.A);
+        tracker.RecordBreakerOpen("d1", "d1", DateTime.UtcNow);
+        tracker.RecordBreakerOpen("d2", "d2", DateTime.UtcNow);
+
+        factory.Create("d1", "Modbus");
+
+        tracker.TryGet("d1", "d1").ShouldBeNull(
+            "the respawn discarded the pipeline whose breaker this snapshot described");
+        tracker.TryGet("d2", "d2")!.IsBreakerOpen.ShouldBeTrue("other instances must be untouched");
+    }
+
    /// <summary>
    ///     End-to-end proof the per-instance override REACHES execution: a ResilienceConfig that sets
    ///     Read's retryCount to 0 must suppress the tier-A Read retry — a failing Read is attempted once,
@@ -331,6 +331,19 @@ public sealed class DriverResilienceOptionsParserTests
        diag.ShouldContain("AlarmAcknowledge never retries");
    }

+    /// <summary>An Initialize retryCount override is forced to 0 with a diagnostic — the
+    /// DriverInstanceActor reconnect timer is the retry loop, so pipeline retries would stack.</summary>
+    [Fact]
+    public void InitializeRetryOverride_IsForcedToZero_WithDiagnostic()
+    {
+        var options = DriverResilienceOptionsParser.ParseOrDefaults(
+            DriverTier.A, "{\"capabilityPolicies\":{\"Initialize\":{\"retryCount\":3}}}", out var diag);
+
+        options.Resolve(DriverCapability.Initialize).RetryCount.ShouldBe(0);
+        diag.ShouldNotBeNull();
+        diag.ShouldContain("Initialize never retries");
+    }
+
    /// <summary>A Read (idempotent) retryCount override is still honored — the invariant is write-shaped only.</summary>
    [Fact]
    public void ReadRetryOverride_StillHonored()
@@ -78,6 +78,39 @@ public sealed class DriverResilienceOptionsTests
        defaults[DriverCapability.AlarmAcknowledge].RetryCount.ShouldBe(0);
    }

+    /// <summary>
+    ///     Initialize never retries by default — the <c>DriverInstanceActor</c> reconnect timer IS the
+    ///     retry loop; pipeline retries would stack inside each timer tick.
+    /// </summary>
+    /// <param name="tier">The driver tier to test.</param>
+    [Theory]
+    [InlineData(DriverTier.A)]
+    [InlineData(DriverTier.B)]
+    [InlineData(DriverTier.C)]
+    public void Initialize_NeverRetries_ByDefault(DriverTier tier)
+    {
+        var defaults = DriverResilienceOptions.GetTierDefaults(tier);
+        defaults[DriverCapability.Initialize].RetryCount.ShouldBe(0);
+    }
+
+    /// <summary>
+    ///     The Initialize breaker threshold must be reachable at the reconnect cadence (R2-10 T10):
+    ///     the breaker samples over 30 s and the reconnect timer fires every 10 s, so at most THREE
+    ///     init attempts fit one sampling window. A threshold above 3 can never open on connect
+    ///     failures — the exact silent-inertness this capability exists to fix — so the default is
+    ///     pinned at 2 (Polly's minimum enabled throughput). Tier C stays breaker-less, consistent
+    ///     with every other Tier C capability (the supervisor's process-level breaker owns it).
+    /// </summary>
+    [Theory]
+    [InlineData(DriverTier.A, 2)]
+    [InlineData(DriverTier.B, 2)]
+    [InlineData(DriverTier.C, 0)]
+    public void Initialize_BreakerThreshold_IsReachableAtTheReconnectCadence(DriverTier tier, int expectedThreshold)
+    {
+        var defaults = DriverResilienceOptions.GetTierDefaults(tier);
+        defaults[DriverCapability.Initialize].BreakerFailureThreshold.ShouldBe(expectedThreshold);
+    }
+
    /// <summary>Verifies that idempotent capabilities retry by default.</summary>
    /// <param name="tier">The driver tier to test.</param>
    /// <param name="capability">The driver capability to test.</param>
@@ -315,6 +315,47 @@ public sealed class DriverResiliencePipelineBuilderTests
        snap.IsBreakerOpen.ShouldBeFalse("the breaker must read closed after a successful recovery probe");
    }

+    /// <summary>
+    ///     R2-10 T10 closure: with the TIER DEFAULTS (no override), consecutive Initialize failures
+    ///     open the breaker — and a subsequent attempt fast-fails WITHOUT reaching the call site.
+    ///     The load-bearing half is the threshold: at the 10 s reconnect cadence at most three init
+    ///     attempts fit the 30 s sampling window, so this test drives exactly the tier-default
+    ///     threshold of failures and demands an open breaker — it goes red if anyone raises the
+    ///     Initialize threshold back into never-opens territory (5), the silent inertness the
+    ///     live gate proved.
+    /// </summary>
+    [Fact]
+    public async Task Initialize_BreakerOpens_AtTierDefaults_AndFastFailsTheNextAttempt()
+    {
+        var tracker = new DriverResilienceStatusTracker();
+        var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
+        var pipeline = builder.GetOrCreate("drv-init", "drv-init", DriverCapability.Initialize, TierAOptions);
+
+        var threshold = TierAOptions.Resolve(DriverCapability.Initialize).BreakerFailureThreshold;
+        for (var i = 0; i < threshold; i++)
+        {
+            await Should.ThrowAsync<InvalidOperationException>(async () =>
+                await pipeline.ExecuteAsync(async _ =>
+                {
+                    await Task.Yield();
+                    throw new InvalidOperationException("connect refused");
+                }));
+        }
+
+        tracker.TryGet("drv-init", "drv-init")!.IsBreakerOpen.ShouldBeTrue(
+            "the tier-default threshold of consecutive connect failures must open the Initialize breaker");
+
+        // The open breaker fast-fails the next attempt without invoking the connect at all.
+        var reached = false;
+        await Should.ThrowAsync<BrokenCircuitException>(async () =>
+            await pipeline.ExecuteAsync(_ =>
+            {
+                reached = true;
+                return ValueTask.CompletedTask;
+            }));
+        reached.ShouldBeFalse("an open breaker must not let the connect attempt through");
+    }
+
    /// <summary>Verifies that the tracker isolates counters per host.</summary>
    [Fact]
    public async Task Tracker_IsolatesCounters_PerHost()
@@ -90,6 +90,27 @@ public sealed class DriverResilienceStatusTrackerTests
        tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
    }

+    /// <summary>
+    ///     Invalidate drops every entry for one instance and leaves other instances alone. Without
+    ///     this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale
+    ///     "Breaker OPEN" snapshot: the respawn discards the pipelines whose <c>OnClosed</c> is the
+    ///     only thing that stamps the close, so nothing can ever clear the flag again.
+    /// </summary>
+    [Fact]
+    public void Invalidate_DropsTheInstance_LeavesOthers()
+    {
+        var tracker = new DriverResilienceStatusTracker();
+        tracker.RecordBreakerOpen("drv", "host-a", Now);
+        tracker.RecordBreakerOpen("drv", "host-b", Now);
+        tracker.RecordBreakerOpen("other", "host-a", Now);
+
+        tracker.Invalidate("drv");
+
+        tracker.TryGet("drv", "host-a").ShouldBeNull();
+        tracker.TryGet("drv", "host-b").ShouldBeNull();
+        tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue();
+    }
+
    /// <summary>Verifies that RecordRecycle populates the LastRecycleUtc timestamp.</summary>
    [Fact]
    public void RecordRecycle_Populates_LastRecycleUtc()