fix(runtime): a removed driver's late writes no longer resurrect its resilience/telemetry state (#609)

DriverHostActor.StopChild released the tracker (Invalidate) and evicted the hub synchronously, but Context.Stop is asynchronous: the stopping child's in-flight SubscribeBulk completed through its capability invoker AFTER the release, and its RecordCallStart/RecordSuccess/RecordCallComplete re-created the tracker entry, which DriverResilienceStatusPublisherService then re-emitted every tick forever. Confirmed with a temporary stack capture in the tracker (removed): every post-Release write came from DriverInstanceActor.HandleSubscribeAsync -> CapabilityInvoker.ExecuteAsync; the test failed whenever it landed before the second publisher tick. - DriverResilienceStatusTracker.Retire (Release) drops the id's snapshots and ignores every later Record* for it; one guarded write path re-checks after the upsert so a write racing Retire cannot survive it. Reinstate (called by DriverCapabilityInvokerFactory.Create) lifts it, so the same id re-added in a later deployment records again. - TelemetryLocalHub.Evict now also holds the id (late Health/Resilience dropped, neither cached nor fanned out); ITelemetryLocalHub.Readmit (default no-op), called at the top of SpawnChild, lifts it. - The removal test is now deterministic: an invoker decorator holds the child's subscribe inside the real invoker's bracket across the removal and signals when its late writes are done (positive control) — red 5/5 without the fix, green 20/20 with it. New unit tests for the tracker guard (incl. a Retire race), the factory Release/Create pair, the hub hold/readmit, and a re-add-same-id host test. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-08 09:07:11 -04:00
parent 5c0623ad2b
commit 4a10ea1e1c
10 changed files with 418 additions and 42 deletions
@@ -80,6 +80,9 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
        // The tracker's snapshots describe the breakers inside the pipelines just discarded, so they go
        // too — a retained "Breaker OPEN" from a pre-respawn outage could never self-correct (only the
        // discarded pipeline's OnClosed stamps the close) and would show a false OPEN chip forever.
+        // Reinstate first: an instance id removed by an earlier deployment (Release ⇒ Retire) and now re-added
+        // must record again (#609).
+        _statusTracker.Reinstate(driverInstanceId);
        _statusTracker.Invalidate(driverInstanceId);

        return new CapabilityInvoker(
@@ -96,10 +99,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
    ///     #609: without this the tracker kept a removed instance's snapshots forever (only <see cref="Create"/>
    ///     invalidated, and a removed instance is never re-created), and the resilience publisher re-emitted
    ///     them every tick — undoing the host's telemetry-hub eviction within one interval.
+    ///     <para>The tracker <b>retires</b> the instance rather than merely invalidating it: the host stops the
+    ///     child asynchronously, so a capability call it still had in flight completes through the released
+    ///     invoker afterwards, and its call-start/success/complete writes re-created the entry (observed: the
+    ///     child's in-flight SubscribeBulk). A retired id ignores those writes until a later <see cref="Create"/>
+    ///     for the same id reinstates it.</para>
    /// </remarks>
    public void Release(string driverInstanceId)
    {
        _builder.Invalidate(driverInstanceId);
-        _statusTracker.Invalidate(driverInstanceId);
+        _statusTracker.Retire(driverInstanceId);
    }
 }
@@ -22,14 +22,38 @@ public sealed class DriverResilienceStatusTracker
 {
    private readonly ConcurrentDictionary<StatusKey, ResilienceStatusSnapshot> _status = new();

+    // Instances retired by Retire (removed from the deployed plan) and not since Reinstated (#609). Every
+    // Record* for one of these is ignored: the removed instance's child stops ASYNCHRONOUSLY, so a capability
+    // call it still had in flight completes through its invoker after the host's release and would otherwise
+    // re-create the entry — which the resilience publisher then re-emits every tick, forever. One entry per
+    // distinct removed id, so the set stays as small as the driver churn that fills it.
+    private readonly ConcurrentDictionary<string, byte> _retired = new(StringComparer.Ordinal);
+
+    /// <summary>
+    ///     The single write path for every <c>Record*</c>. A write for a retired instance is dropped; the
+    ///     post-write re-check closes the race with a concurrent <see cref="Retire"/> — a writer that passed the
+    ///     first check just before the retire and lands its upsert just after the retire's sweep removes its own
+    ///     write again, so no interleaving resurrects the instance.
+    /// </summary>
+    private void Upsert(
+        string driverInstanceId,
+        string hostName,
+        Func<StatusKey, ResilienceStatusSnapshot> add,
+        Func<StatusKey, ResilienceStatusSnapshot, ResilienceStatusSnapshot> update)
+    {
+        if (_retired.ContainsKey(driverInstanceId)) return;
+        var key = new StatusKey(driverInstanceId, hostName);
+        _status.AddOrUpdate(key, add, update);
+        if (_retired.ContainsKey(driverInstanceId)) _status.TryRemove(key, out _);
+    }
+
    /// <summary>Record a Polly pipeline failure for <paramref name="hostName"/>.</summary>
    /// <param name="driverInstanceId">The driver instance identifier.</param>
    /// <param name="hostName">The host name.</param>
    /// <param name="utcNow">The UTC timestamp of the failure event.</param>
    public void RecordFailure(string driverInstanceId, string hostName, DateTime utcNow)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { ConsecutiveFailures = 1, LastSampledUtc = utcNow },
            (_, existing) => existing with
            {
@@ -44,8 +68,7 @@ public sealed class DriverResilienceStatusTracker
    /// <param name="utcNow">The UTC timestamp of the success event.</param>
    public void RecordSuccess(string driverInstanceId, string hostName, DateTime utcNow)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { ConsecutiveFailures = 0, LastSampledUtc = utcNow },
            (_, existing) => existing with
            {
@@ -60,8 +83,7 @@ public sealed class DriverResilienceStatusTracker
    /// <param name="utcNow">The UTC timestamp of the breaker open event.</param>
    public void RecordBreakerOpen(string driverInstanceId, string hostName, DateTime utcNow)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow },
            (_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow });
    }
@@ -77,8 +99,7 @@ public sealed class DriverResilienceStatusTracker
    /// <param name="utcNow">The UTC timestamp of the breaker close event.</param>
    public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow },
            (_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow });
    }
@@ -104,14 +125,39 @@ public sealed class DriverResilienceStatusTracker
        return removed;
    }

+    /// <summary>
+    ///     Drop every snapshot for a driver instance that has been REMOVED from the deployed plan, and ignore
+    ///     every later write for it until <see cref="Reinstate"/> (#609). Called by
+    ///     <c>DriverCapabilityInvokerFactory.Release</c>. Unlike <see cref="Invalidate"/> (a respawn, whose new
+    ///     invoker records straight away), the removed instance's child is still stopping — a call it had in
+    ///     flight finishes after this returns, and without the hold its RecordCallStart / RecordSuccess /
+    ///     RecordCallComplete re-create the entry the publisher then re-emits forever.
+    /// </summary>
+    /// <param name="driverInstanceId">The removed driver instance.</param>
+    /// <returns>The number of snapshots removed.</returns>
+    public int Retire(string driverInstanceId)
+    {
+        // Retire BEFORE sweeping: a write racing the sweep either lands first (and is swept) or re-checks after
+        // its upsert and removes itself (see Upsert).
+        _retired[driverInstanceId] = 0;
+        return Invalidate(driverInstanceId);
+    }
+
+    /// <summary>
+    ///     Lift an earlier <see cref="Retire"/> so the instance records again — the same driver instance id
+    ///     re-added in a later deployment. Called by <c>DriverCapabilityInvokerFactory.Create</c>; a no-op for
+    ///     an id that was never retired.
+    /// </summary>
+    /// <param name="driverInstanceId">The driver instance being (re)created.</param>
+    public void Reinstate(string driverInstanceId) => _retired.TryRemove(driverInstanceId, out _);
+
    /// <summary>Record a process recycle event (Tier C only).</summary>
    /// <param name="driverInstanceId">The driver instance identifier.</param>
    /// <param name="hostName">The host name.</param>
    /// <param name="utcNow">The UTC timestamp of the recycle event.</param>
    public void RecordRecycle(string driverInstanceId, string hostName, DateTime utcNow)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { LastRecycleUtc = utcNow, LastSampledUtc = utcNow },
            (_, existing) => existing with { LastRecycleUtc = utcNow, LastSampledUtc = utcNow });
    }
@@ -124,8 +170,7 @@ public sealed class DriverResilienceStatusTracker
    /// <param name="utcNow">The UTC timestamp of the footprint update.</param>
    public void RecordFootprint(string driverInstanceId, string hostName, long baselineBytes, long currentBytes, DateTime utcNow)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot
            {
                BaselineFootprintBytes = baselineBytes,
@@ -150,8 +195,7 @@ public sealed class DriverResilienceStatusTracker
    /// <param name="hostName">The host name.</param>
    public void RecordCallStart(string driverInstanceId, string hostName)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { CurrentInFlight = 1 },
            (_, existing) => existing with { CurrentInFlight = existing.CurrentInFlight + 1 });
    }
@@ -161,8 +205,7 @@ public sealed class DriverResilienceStatusTracker
    /// <param name="hostName">The host name.</param>
    public void RecordCallComplete(string driverInstanceId, string hostName)
    {
-        var key = new StatusKey(driverInstanceId, hostName);
-        _status.AddOrUpdate(key,
+        Upsert(driverInstanceId, hostName,
            _ => new ResilienceStatusSnapshot { CurrentInFlight = 0 }, // start-without-complete shouldn't happen; clamp to 0
            (_, existing) => existing with { CurrentInFlight = Math.Max(0, existing.CurrentInFlight - 1) });
    }