fix(runtime): lifecycle housekeeping — reprovision stamp, Stale drops, off-thread DB probe, tombstone sweep (A4)

- DriverHostActor.StopChild removes the driver's _lastReprovisionUtc stamp
  (and its deferred-reprovision timer): the map grew per removed id and a
  re-added driver inherited a backoff it never earned (02/L2).
- Stale now registers explicit drops for AttributeValueBatchPublished,
  AttributeAlarmPublished, ArtifactForApply and ApplyWedged instead of
  letting them hit Unhandled (02/H5).
- DbHealthProbeActor runs its SELECT on the thread pool and pipes the
  outcome back (single-flight), with a 2 s command timeout on relational
  providers, so a hung SQL Server no longer pins the actor thread and the
  cached GetStatus answer stays reachable (02/P7).
- TelemetryLocalHub eviction tombstones are timestamped and swept past a
  10 min TTL on the next Evict; a TimeProvider ctor is the test seam (02/L4).
- Tests: ValuePublishes.Single helper replaces the per-value
  AttributeValuePublished in CalculationDependencyFlowTests (02/U4, test
  half; the production receive is owned by Lane E).
- R05/8 audit: PollGroupEngine.Unsubscribe has no production caller; every
  driver already uses UnsubscribeAsync (no change).

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -2489,6 +2489,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         // A child connectivity transition while the host is Stale has no live address space to annotate — drop
         // it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
         Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
+        // Likewise a child's value / alarm burst: Stale is entered from Bootstrap's catch before any child is
+        // spawned, so none should arrive — but the orphan-replay BeginDirectApply in that same try block has
+        // already started a pipe-back before the catch can run, so "cannot happen" rests on ordering. Explicit
+        // drops, so a burst that does land is a logged drop rather than an Unhandled dead-letter.
+        Receive<DriverInstanceActor.AttributeValueBatchPublished>(m =>
+            _log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)",
+                _localNode, m.Items.Count, m.DriverInstanceId));
+        Receive<DriverInstanceActor.AttributeAlarmPublished>(m =>
+            _log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale",
+                _localNode, m.Args.ConditionId, m.DriverInstanceId));
+        // A late artifact pipe-back from the apply that was in flight when the host fell Stale: that apply is
+        // lost (the coordinator's deadline and the boot-time orphan replay recover it). Warning, not Debug —
+        // an apply outcome is being discarded.
+        Receive<ArtifactForApply>(m =>
+            _log.Warning("DriverHost {Node}: dropping the artifact for {Id} (rev {Rev}) that arrived while Stale; the apply is abandoned",
+                _localNode, m.DeploymentId, m.Revision));
+        Receive<ApplyWedged>(m =>
+            _log.Debug("DriverHost {Node}: late apply-wedge tick for {Id} while Stale; dropping", _localNode, m.DeploymentId));
         // Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
         // on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
         Receive<CommsRecoveryAudit>(_ => { });
@@ -3771,6 +3789,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         // id lifts it (the tracker via _invokerFactory.Create, the hub via Readmit).
         _invokerFactory.Release(driverInstanceId);
         _telemetryHub?.Evict(driverInstanceId);
+        // The reprovision rate-limit stamp belongs to the incarnation history that just ended. Kept, it grew the
+        // map by one entry per id ever reprovisioned-then-removed AND made a later re-add of the same id inherit a
+        // backoff it never earned (its first crash was deferred as a "crash loop").
+        _lastReprovisionUtc.Remove(driverInstanceId);
+        Timers.Cancel(ReprovisionTimerKey(driverInstanceId));
         _log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
     }
 

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Health/DbHealthProbeActor.cs

@@ -7,21 +7,37 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Health;
 
 /// <summary>
 /// Single-flight cached health probe against the ConfigDb. Reads cached state via
-/// <c>Ask&lt;DbHealthStatus&gt;</c>; a single SELECT 1 runs at most every <c>RefreshInterval</c>.
+/// <c>Ask&lt;DbHealthStatus&gt;</c>; a single SELECT runs at most every <c>RefreshInterval</c>.
 /// Consumed by both the host's <c>/health/ready</c> endpoint and
 /// <c>RedundancyStateActor</c>'s stale calc.
 /// </summary>
+/// <remarks>
+///     The probe runs OFF the mailbox (a thread-pool task that pipes its outcome back), with a bounded
+///     command timeout. It used to run synchronously on the actor thread, relying on SqlClient's implicit
+///     15 s connect / 30 s command timeouts — so a hanging SQL Server pinned a dispatcher thread for up to
+///     30 s per tick, later ticks queued behind it, and the cached <see cref="GetStatus"/> answer was
+///     unreachable for exactly as long as the database was. Single-flight: a tick that lands while a probe
+///     is still running is skipped rather than stacked.
+/// </remarks>
 public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
 {
     public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(5);
 
+    /// <summary>Bound on the probe's SQL command, relational providers only (the in-memory provider the
+    /// actor tests run against has no command timeout).</summary>
+    public static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(2);
+
     public sealed class GetStatus { public static readonly GetStatus Instance = new(); private GetStatus() { } }
     public sealed record DbHealthStatus(bool Reachable, DateTime AsOfUtc, string? LastError);
     public sealed class Tick { public static readonly Tick Instance = new(); private Tick() { } }
 
+    /// <summary>Piped back by the off-thread probe with its outcome.</summary>
+    private sealed record ProbeCompleted(DbHealthStatus Status);
+
     private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
     private readonly ILoggingAdapter _log = Context.GetLogger();
     private DbHealthStatus _last = new(false, DateTime.MinValue, "not probed yet");
+    private bool _probeInFlight;
 
     /// <summary>Gets or sets the timer scheduler for periodic health probes.</summary>
     public ITimerScheduler Timers { get; set; } = null!;
@@ -38,28 +54,54 @@ public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
     {
         _dbFactory = dbFactory;
         Receive<GetStatus>(_ => Sender.Tell(_last));
-        Receive<Tick>(_ => RunProbe());
+        Receive<Tick>(_ => StartProbe());
+        Receive<ProbeCompleted>(msg =>
+        {
+            _probeInFlight = false;
+            _last = msg.Status;
+            if (!msg.Status.Reachable)
+                _log.Warning("DbHealthProbe: probe failed: {Error}", msg.Status.LastError);
+        });
     }
 
     /// <inheritdoc />
     protected override void PreStart()
     {
-        RunProbe();
+        StartProbe();
         Timers.StartPeriodicTimer("probe", Tick.Instance, RefreshInterval);
     }
 
-    private void RunProbe()
+    private void StartProbe()
+    {
+        if (_probeInFlight)
+        {
+            _log.Debug("DbHealthProbe: previous probe still running; skipping this tick");
+            return;
+        }
+
+        _probeInFlight = true;
+        var factory = _dbFactory;
+        // Never touches actor state; the outcome comes back as a message.
+        Task.Run(() => Probe(factory)).PipeTo(
+            Self,
+            success: status => new ProbeCompleted(status),
+            failure: ex => new ProbeCompleted(new DbHealthStatus(false, DateTime.UtcNow, ex.Message)));
+    }
+
+    private static DbHealthStatus Probe(IDbContextFactory<OtOpcUaConfigDbContext> factory)
     {
         try
         {
-            using var db = _dbFactory.CreateDbContext();
+            using var db = factory.CreateDbContext();
+            // Local to this lease — the pool resets it on return (see DriverHostActor.LoadArtifactBlob).
+            if (db.Database.IsRelational())
+                db.Database.SetCommandTimeout(CommandTimeout);
             _ = db.Deployments.AsNoTracking().Take(1).ToList();
-            _last = new DbHealthStatus(true, DateTime.UtcNow, null);
+            return new DbHealthStatus(true, DateTime.UtcNow, null);
         }
         catch (Exception ex)
         {
-            _last = new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
-            _log.Warning(ex, "DbHealthProbe: probe failed");
+            return new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
         }
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/TelemetryLocalHub.cs

@@ -27,7 +27,27 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
 /// </remarks>
 public sealed class TelemetryLocalHub : ITelemetryLocalHub
 {
+    /// <summary>
+    ///     How long an eviction tombstone (see <see cref="_evicted"/>) is kept. It only has to outlive the
+    ///     removed child's last late write — an in-flight capability call that completes after
+    ///     <c>StopChild</c>, bounded by the longest capability timeout (seconds) plus the child's own stop —
+    ///     so ten minutes is a wide margin. Without a sweep a driver removed for good kept its id forever.
+    /// </summary>
+    public static readonly TimeSpan EvictionTombstoneTtl = TimeSpan.FromMinutes(10);
+
     private readonly object _gate = new();
+    private readonly TimeProvider _time;
+
+    /// <summary>Initializes a hub on the system clock.</summary>
+    public TelemetryLocalHub() : this(TimeProvider.System) { }
+
+    /// <summary>Initializes a hub on <paramref name="time"/> — the tombstone-sweep test seam.</summary>
+    /// <param name="time">The clock the eviction tombstones are stamped and swept by.</param>
+    public TelemetryLocalHub(TimeProvider time)
+    {
+        ArgumentNullException.ThrowIfNull(time);
+        _time = time;
+    }
 
     // Plain Dictionary, NOT ConcurrentDictionary: every access is already under _gate, and
     // ConcurrentDictionary.Values would materialize a fresh List (and take its own internal lock) on
@@ -42,9 +62,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
     private readonly ConcurrentDictionary<string, TelemetryItem.Health> _healthCache = new();
     private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
 
-    // Instances evicted and not since readmitted. Their late Health/Resilience items are dropped, so a
-    // removed driver's still-stopping child cannot re-prime the cache it was just evicted from. Under _gate.
-    private readonly HashSet<string> _evicted = new(StringComparer.Ordinal);
+    // Instances evicted and not since readmitted, stamped with the eviction time. Their late Health/Resilience
+    // items are dropped, so a removed driver's still-stopping child cannot re-prime the cache it was just
+    // evicted from. Swept past EvictionTombstoneTtl on the next Evict (never on the Emit hot path). Under _gate.
+    private readonly Dictionary<string, DateTimeOffset> _evicted = new(StringComparer.Ordinal);
 
     /// <inheritdoc />
     public void Emit(TelemetryItem item)
@@ -55,9 +76,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
         {
             switch (item)
             {
-                case TelemetryItem.Health h when _evicted.Contains(h.E.DriverInstanceId):
+                case TelemetryItem.Health h when _evicted.ContainsKey(h.E.DriverInstanceId):
                     return;
-                case TelemetryItem.Resilience r when _evicted.Contains(r.E.DriverInstanceId):
+                case TelemetryItem.Resilience r when _evicted.ContainsKey(r.E.DriverInstanceId):
                     return;
                 case TelemetryItem.Health h:
                     _healthCache[h.E.DriverInstanceId] = h;
@@ -80,7 +101,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
 
         lock (_gate)
         {
-            _evicted.Add(driverInstanceId);
+            var now = _time.GetUtcNow();
+            SweepExpiredTombstones(now);
+            _evicted[driverInstanceId] = now;
             _healthCache.TryRemove(driverInstanceId, out _);
             foreach (var key in _resilienceCache.Keys)
                 if (string.Equals(key.InstanceId, driverInstanceId, StringComparison.Ordinal))
@@ -97,6 +120,24 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
             _evicted.Remove(driverInstanceId);
     }
 
+    /// <summary>Drops tombstones older than <see cref="EvictionTombstoneTtl"/>. Called under <see cref="_gate"/>
+    /// from <see cref="Evict"/> only — the set is bounded by distinct removed ids, so a linear pass per
+    /// eviction is cheap, and Emit never pays for it.</summary>
+    private void SweepExpiredTombstones(DateTimeOffset now)
+    {
+        if (_evicted.Count == 0) return;
+        List<string>? expired = null;
+        foreach (var (id, evictedAt) in _evicted)
+        {
+            if (now - evictedAt >= EvictionTombstoneTtl)
+                (expired ??= new List<string>()).Add(id);
+        }
+
+        if (expired is null) return;
+        foreach (var id in expired)
+            _evicted.Remove(id);
+    }
+
     /// <inheritdoc />
     public ITelemetrySubscription Subscribe(int boundedCapacity)
     {