feat(resilience): route IDriver.InitializeAsync through the capability invoker (R2-10 T10 closed)

The connect path was the one capability call the resilience pipeline never saw, so a driver that could not connect contributed zero breaker input and the /hosts resilience panel read 'All targets healthy' against a dead endpoint. InitializeAsync now routes through the invoker as DriverCapability.Initialize, keyed on the driver-instance id. - No pipeline retry: the actor's RetryConnect timer is the retry loop (parser forces retryCount overrides to 0, same funnel as write-shaped). - Breaker threshold 2, not the Read-sized 5: the breaker samples 30 s and the reconnect timer fires every 10 s, so at most three attempts fit one window — threshold 5 could never open. Pinned by its own test. - An open breaker fast-fails into the same InitializeFailed path as a real connect failure; the 15 s break window costs at most one extra tick. The best-effort teardown ShutdownAsync stays unwrapped. - Companion fix: DriverCapabilityInvokerFactory.Create now invalidates the instance's tracker snapshots with its pipelines — a respawn during an outage discarded the only breaker whose OnClosed could clear the OPEN flag, stranding a false 'Breaker OPEN' chip forever. Live gate PASSED on docker-dev (image symbol-verified per container): rtu-gate against dead 10.100.0.35:5021 opened its Initialize breaker on both centrals with the red 'BREAKER OPEN' chip on /hosts; a socat 5021→5020 forwarder recovered it (both breakers closed within 1 s, panel back to 'All targets healthy'); healthy MAIN-modbus recorded zero breaker events. Full suite 50/50 green. Evidence: docs/plans/2026-08-07-adminui-followups-and-live-gates.md §7.
2026-08-08 05:45:08 -04:00
parent c9f182b2d7
commit dac7e1885d
18 changed files with 445 additions and 8 deletions
@@ -39,4 +39,15 @@ public enum DriverCapability

    /// <summary><see cref="IHistoryProvider"/> reads (Raw/Processed/AtTime/Events). Retries by default.</summary>
    HistoryRead,
+
+    /// <summary>
+    ///     <c>IDriver.InitializeAsync</c> — connection establishment (R2-10 T10 closure: without this,
+    ///     a driver that cannot connect contributes zero breaker input and the resilience panel stays
+    ///     quiet for the failure operators most care about). Does NOT retry — the
+    ///     <c>DriverInstanceActor</c> reconnect timer is the retry loop, and pipeline-level retries
+    ///     would stack inside each timer tick. The tier-default breaker threshold is deliberately low
+    ///     (2): at the 10 s reconnect cadence at most three attempts fit the breaker's 30 s sampling
+    ///     window, so a Read-sized threshold of 5 could never open on connect failures.
+    /// </summary>
+    Initialize,
 }
@@ -77,6 +77,10 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
        // Cleanup (no longer the correctness mechanism): drop this instance's cached pipelines across all
        // generations so lingering old-generation entries don't accumulate. No-op on first spawn.
        _builder.Invalidate(driverInstanceId);
+        // The tracker's snapshots describe the breakers inside the pipelines just discarded, so they go
+        // too — a retained "Breaker OPEN" from a pre-respawn outage could never self-correct (only the
+        // discarded pipeline's OnClosed stamps the close) and would show a false OPEN chip forever.
+        _statusTracker.Invalidate(driverInstanceId);

        return new CapabilityInvoker(
            _builder,
@@ -82,6 +82,11 @@ public sealed record DriverResilienceOptions
                [DriverCapability.AlarmSubscribe]   = new(TimeoutSeconds: 5,  RetryCount: 3, BreakerFailureThreshold: 5),
                [DriverCapability.AlarmAcknowledge] = new(TimeoutSeconds: 5,  RetryCount: 0, BreakerFailureThreshold: 5),
                [DriverCapability.HistoryRead]      = new(TimeoutSeconds: 30, RetryCount: 2, BreakerFailureThreshold: 5),
+                // Initialize: no retry (the DriverInstanceActor reconnect timer is the retry loop) and a
+                // threshold of 2, NOT the Read-sized 5 — the breaker samples over 30 s and the reconnect
+                // timer fires every 10 s, so at most three init attempts fit one window; a threshold of 5
+                // could never open on connect failures (R2-10 T10).
+                [DriverCapability.Initialize]       = new(TimeoutSeconds: 5,  RetryCount: 0, BreakerFailureThreshold: 2),
            },
            DriverTier.B => new Dictionary<DriverCapability, CapabilityPolicy>
            {
@@ -93,6 +98,8 @@ public sealed record DriverResilienceOptions
                [DriverCapability.AlarmSubscribe]   = new(TimeoutSeconds: 8,  RetryCount: 3, BreakerFailureThreshold: 5),
                [DriverCapability.AlarmAcknowledge] = new(TimeoutSeconds: 8,  RetryCount: 0, BreakerFailureThreshold: 5),
                [DriverCapability.HistoryRead]      = new(TimeoutSeconds: 60, RetryCount: 2, BreakerFailureThreshold: 5),
+                // See the Tier A Initialize note: threshold 2 is cadence-reachability, not a tuning choice.
+                [DriverCapability.Initialize]       = new(TimeoutSeconds: 8,  RetryCount: 0, BreakerFailureThreshold: 2),
            },
            DriverTier.C => new Dictionary<DriverCapability, CapabilityPolicy>
            {
@@ -104,6 +111,8 @@ public sealed record DriverResilienceOptions
                [DriverCapability.AlarmSubscribe]   = new(TimeoutSeconds: 15,  RetryCount: 1, BreakerFailureThreshold: 0),
                [DriverCapability.AlarmAcknowledge] = new(TimeoutSeconds: 15,  RetryCount: 0, BreakerFailureThreshold: 0),
                [DriverCapability.HistoryRead]      = new(TimeoutSeconds: 120, RetryCount: 1, BreakerFailureThreshold: 0),
+                // Tier C stays breaker-less like every other Tier C capability (process-level supervision).
+                [DriverCapability.Initialize]       = new(TimeoutSeconds: 15,  RetryCount: 0, BreakerFailureThreshold: 0),
            },
            _ => throw new ArgumentOutOfRangeException(nameof(tier), tier, $"No default policy table defined for tier {tier}."),
        };
@@ -158,6 +158,17 @@ public static class DriverResilienceOptionsParser
            retry = 0;
        }

+        // Initialize's retry loop is the DriverInstanceActor reconnect timer, not the pipeline —
+        // pipeline retries would stack inside each timer tick (and, worse, stretch a single init
+        // attempt past the timer interval so attempts overlap). Same funnel-enforced invariant
+        // shape as the write-shaped rule above, different rationale.
+        if (capability == DriverCapability.Initialize && retry > 0)
+        {
+            AppendDiagnostic(ref diagnostic,
+                $"{capability} never retries (the reconnect timer is the retry loop); retryCount override ignored.");
+            retry = 0;
+        }
+
        var breaker = merged.BreakerFailureThreshold;
        if (breaker < ResiliencePolicyRanges.MinBreakerFailureThreshold)
        {
@@ -83,6 +83,27 @@ public sealed class DriverResilienceStatusTracker
            (_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow });
    }

+    /// <summary>
+    ///     Drop every snapshot for one driver instance. Called by
+    ///     <c>DriverCapabilityInvokerFactory.Create</c> alongside
+    ///     <c>DriverResiliencePipelineBuilder.Invalidate</c>: the pipelines carry the actual breaker
+    ///     state, so once they are discarded on a respawn, a retained snapshot describes a breaker
+    ///     that no longer exists — and a stale "Breaker OPEN" could never self-correct, because only
+    ///     the discarded pipeline's <c>OnClosed</c> hook stamps the close.
+    /// </summary>
+    /// <param name="driverInstanceId">The driver instance whose snapshots should be dropped.</param>
+    /// <returns>The number of snapshots removed.</returns>
+    public int Invalidate(string driverInstanceId)
+    {
+        var removed = 0;
+        foreach (var key in _status.Keys)
+        {
+            if (key.DriverInstanceId == driverInstanceId && _status.TryRemove(key, out _))
+                removed++;
+        }
+        return removed;
+    }
+
    /// <summary>Record a process recycle event (Tier C only).</summary>
    /// <param name="driverInstanceId">The driver instance identifier.</param>
    /// <param name="hostName">The host name.</param>
@@ -21,7 +21,7 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
 public sealed class ResilienceFormModel
 {
    public static readonly string[] Capabilities =
-        ["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead"];
+        ["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead", "Initialize"];

    // RecycleIntervalSeconds is GONE (Gitea #522) — the scheduled-recycle machinery it authored was
    // deleted, so the field configured nothing. It is deliberately NOT explicitly stripped from stored
@@ -687,6 +687,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
        var log = _log;
        var driverInstanceId = _driverInstanceId;
        var driver = _driver;
+        var invoker = _invoker;
        _ = Task.Run(async () =>
        {
            try
@@ -695,6 +696,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
                {
                    try
                    {
+                        // Deliberately UNWRAPPED: teardown is best-effort cleanup of an already-dead
+                        // transport — a shutdown failure must not count toward the Initialize breaker.
                        await driver.ShutdownAsync(CancellationToken.None);
                    }
                    catch (Exception ex)
@@ -704,7 +707,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
                            "initialising anyway", driverInstanceId);
                    }
                }
-                await driver.InitializeAsync(driverConfigJson, CancellationToken.None);
+                // R2-10 T10: the connect path was the ONE capability call the invoker never saw, so a
+                // driver that could not connect contributed zero breaker input and the resilience panel
+                // stayed quiet for a dead endpoint. Keyed on the driver-instance id (init spans the
+                // whole driver, like a bulk subscribe). The Initialize policy never retries — this
+                // actor's RetryConnect timer is the retry loop — and its pipeline timeout now bounds a
+                // hanging connect via the ct the call site must honour. An open breaker fast-fails the
+                // attempt with BrokenCircuitException, which lands in the catch below exactly like a
+                // real connect failure: InitializeFailed → keep retrying on the timer. The 15 s break
+                // window is shorter than two 10 s timer ticks, so recovery costs at most one extra tick.
+                await invoker.ExecuteAsync(
+                    DriverCapability.Initialize,
+                    driverInstanceId,
+                    async ct => await driver.InitializeAsync(driverConfigJson, ct),
+                    CancellationToken.None);
                self.Tell(new InitializeSucceeded(generation));
            }
            catch (Exception ex)