fix(runtime): lifecycle housekeeping — reprovision stamp, Stale drops, off-thread DB probe, tombstone sweep (A4)

- DriverHostActor.StopChild removes the driver's _lastReprovisionUtc stamp
  (and its deferred-reprovision timer): the map grew per removed id and a
  re-added driver inherited a backoff it never earned (02/L2).
- Stale now registers explicit drops for AttributeValueBatchPublished,
  AttributeAlarmPublished, ArtifactForApply and ApplyWedged instead of
  letting them hit Unhandled (02/H5).
- DbHealthProbeActor runs its SELECT on the thread pool and pipes the
  outcome back (single-flight), with a 2 s command timeout on relational
  providers, so a hung SQL Server no longer pins the actor thread and the
  cached GetStatus answer stays reachable (02/P7).
- TelemetryLocalHub eviction tombstones are timestamped and swept past a
  10 min TTL on the next Evict; a TimeProvider ctor is the test seam (02/L4).
- Tests: ValuePublishes.Single helper replaces the per-value
  AttributeValuePublished in CalculationDependencyFlowTests (02/U4, test
  half; the production receive is owned by Lane E).
- R05/8 audit: PollGroupEngine.Unsubscribe has no production caller; every
  driver already uses UnsubscribeAsync (no change).

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:36:12 -04:00
parent 6e2febbf3d
commit 79d6b0a4b2
11 changed files with 460 additions and 26 deletions
@@ -2489,6 +2489,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
// Likewise a child's value / alarm burst: Stale is entered from Bootstrap's catch before any child is
// spawned, so none should arrive — but the orphan-replay BeginDirectApply in that same try block has
// already started a pipe-back before the catch can run, so "cannot happen" rests on ordering. Explicit
// drops, so a burst that does land is a logged drop rather than an Unhandled dead-letter.
Receive<DriverInstanceActor.AttributeValueBatchPublished>(m =>
_log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)",
_localNode, m.Items.Count, m.DriverInstanceId));
Receive<DriverInstanceActor.AttributeAlarmPublished>(m =>
_log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale",
_localNode, m.Args.ConditionId, m.DriverInstanceId));
// A late artifact pipe-back from the apply that was in flight when the host fell Stale: that apply is
// lost (the coordinator's deadline and the boot-time orphan replay recover it). Warning, not Debug —
// an apply outcome is being discarded.
Receive<ArtifactForApply>(m =>
_log.Warning("DriverHost {Node}: dropping the artifact for {Id} (rev {Rev}) that arrived while Stale; the apply is abandoned",
_localNode, m.DeploymentId, m.Revision));
Receive<ApplyWedged>(m =>
_log.Debug("DriverHost {Node}: late apply-wedge tick for {Id} while Stale; dropping", _localNode, m.DeploymentId));
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
Receive<CommsRecoveryAudit>(_ => { });
@@ -3771,6 +3789,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// id lifts it (the tracker via _invokerFactory.Create, the hub via Readmit).
_invokerFactory.Release(driverInstanceId);
_telemetryHub?.Evict(driverInstanceId);
// The reprovision rate-limit stamp belongs to the incarnation history that just ended. Kept, it grew the
// map by one entry per id ever reprovisioned-then-removed AND made a later re-add of the same id inherit a
// backoff it never earned (its first crash was deferred as a "crash loop").
_lastReprovisionUtc.Remove(driverInstanceId);
Timers.Cancel(ReprovisionTimerKey(driverInstanceId));
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}