fix(runtime): a removed driver's late writes no longer resurrect its resilience/telemetry state (#609)

DriverHostActor.StopChild released the tracker (Invalidate) and evicted the hub synchronously, but
Context.Stop is asynchronous: the stopping child's in-flight SubscribeBulk completed through its
capability invoker AFTER the release, and its RecordCallStart/RecordSuccess/RecordCallComplete
re-created the tracker entry, which DriverResilienceStatusPublisherService then re-emitted every tick
forever. Confirmed with a temporary stack capture in the tracker (removed): every post-Release write
came from DriverInstanceActor.HandleSubscribeAsync -> CapabilityInvoker.ExecuteAsync; the test failed
whenever it landed before the second publisher tick.

- DriverResilienceStatusTracker.Retire (Release) drops the id's snapshots and ignores every later
  Record* for it; one guarded write path re-checks after the upsert so a write racing Retire cannot
  survive it. Reinstate (called by DriverCapabilityInvokerFactory.Create) lifts it, so the same id
  re-added in a later deployment records again.
- TelemetryLocalHub.Evict now also holds the id (late Health/Resilience dropped, neither cached nor
  fanned out); ITelemetryLocalHub.Readmit (default no-op), called at the top of SpawnChild, lifts it.
- The removal test is now deterministic: an invoker decorator holds the child's subscribe inside the
  real invoker's bracket across the removal and signals when its late writes are done (positive
  control) — red 5/5 without the fix, green 20/20 with it. New unit tests for the tracker guard
  (incl. a Retire race), the factory Release/Create pair, the hub hold/readmit, and a re-add-same-id
  host test.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -3198,6 +3198,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
 
     private void SpawnChild(DriverInstanceSpec spec)
     {
+        // #609: an instance id removed by an earlier deployment (StopChild ⇒ Evict, which also holds back late
+        // items) may come back; lift the hold BEFORE the child exists, since it publishes health as it starts.
+        // The resilience tracker's equivalent is lifted by _invokerFactory.Create below.
+        _telemetryHub?.Readmit(spec.DriverInstanceId);
         var stub = DriverInstanceActor.ShouldStub(spec.DriverType, _localRoles);
         IDriver? driver = null;
         if (!stub)
@@ -3388,7 +3392,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         // #609: the instance left the plan (StopChild is only reached via plan.ToStop), so its cached
         // Health/Resilience snapshots must not replay to new telemetry subscribers. Release the shared
         // resilience state FIRST: the resilience publisher re-emits the tracker's snapshots every tick, so
-        // evicting the hub alone was undone within one interval.
+        // evicting the hub alone was undone within one interval. Context.Stop is asynchronous, so the child can
+        // still finish an in-flight capability call (observed: its SubscribeBulk) and publish health AFTER these
+        // two calls; both therefore HOLD the id — late writes are dropped — until a later SpawnChild of the same
+        // id lifts it (the tracker via _invokerFactory.Create, the hub via Readmit).
         _invokerFactory.Release(driverInstanceId);
         _telemetryHub?.Evict(driverInstanceId);
         _log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/ITelemetryLocalHub.cs

@@ -89,6 +89,16 @@ public interface ITelemetryLocalHub
     ///     Drops the cached Health and Resilience snapshots of a driver instance that has been removed from
     ///     this node, so a later <see cref="Subscribe"/> no longer replays its last-known state. Live
     ///     subscribers are not told; the instance simply stops emitting. Unknown ids are a no-op.
+    ///     <para>The eviction also <b>holds</b> (#609): a removed instance's child stops asynchronously and can
+    ///     still publish after this call (a health snapshot when an in-flight connect or subscribe finishes),
+    ///     so until <see cref="Readmit"/> every later Health/Resilience item for the instance is dropped — not
+    ///     cached and not fanned out.</para>
     /// </summary>
     void Evict(string driverInstanceId);
+
+    /// <summary>
+    ///     Lifts an earlier <see cref="Evict"/> so a driver instance spawned again under the same id (a later
+    ///     deployment re-adding it) publishes normally. A no-op for an id that was never evicted.
+    /// </summary>
+    void Readmit(string driverInstanceId) { }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/TelemetryLocalHub.cs

@@ -42,6 +42,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
     private readonly ConcurrentDictionary<string, TelemetryItem.Health> _healthCache = new();
     private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
 
+    // Instances evicted and not since readmitted (#609). Their late Health/Resilience items are dropped, so a
+    // removed driver's still-stopping child cannot re-prime the cache it was just evicted from. Under _gate.
+    private readonly HashSet<string> _evicted = new(StringComparer.Ordinal);
+
     /// <inheritdoc />
     public void Emit(TelemetryItem item)
     {
@@ -51,6 +55,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
         {
             switch (item)
             {
+                case TelemetryItem.Health h when _evicted.Contains(h.E.DriverInstanceId):
+                    return;
+                case TelemetryItem.Resilience r when _evicted.Contains(r.E.DriverInstanceId):
+                    return;
                 case TelemetryItem.Health h:
                     _healthCache[h.E.DriverInstanceId] = h;
                     break;
@@ -72,6 +80,7 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
 
         lock (_gate)
         {
+            _evicted.Add(driverInstanceId);
             _healthCache.TryRemove(driverInstanceId, out _);
             foreach (var key in _resilienceCache.Keys)
                 if (string.Equals(key.InstanceId, driverInstanceId, StringComparison.Ordinal))
@@ -79,6 +88,15 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
         }
     }
 
+    /// <inheritdoc />
+    public void Readmit(string driverInstanceId)
+    {
+        ArgumentNullException.ThrowIfNull(driverInstanceId);
+
+        lock (_gate)
+            _evicted.Remove(driverInstanceId);
+    }
+
     /// <inheritdoc />
     public ITelemetrySubscription Subscribe(int boundedCapacity)
     {