fix(runtime): lifecycle housekeeping — reprovision stamp, Stale drops, off-thread DB probe, tombstone sweep (A4)

- DriverHostActor.StopChild removes the driver's _lastReprovisionUtc stamp
  (and its deferred-reprovision timer): the map grew per removed id and a
  re-added driver inherited a backoff it never earned (02/L2).
- Stale now registers explicit drops for AttributeValueBatchPublished,
  AttributeAlarmPublished, ArtifactForApply and ApplyWedged instead of
  letting them hit Unhandled (02/H5).
- DbHealthProbeActor runs its SELECT on the thread pool and pipes the
  outcome back (single-flight), with a 2 s command timeout on relational
  providers, so a hung SQL Server no longer pins the actor thread and the
  cached GetStatus answer stays reachable (02/P7).
- TelemetryLocalHub eviction tombstones are timestamped and swept past a
  10 min TTL on the next Evict; a TimeProvider ctor is the test seam (02/L4).
- Tests: ValuePublishes.Single helper replaces the per-value
  AttributeValuePublished in CalculationDependencyFlowTests (02/U4, test
  half; the production receive is owned by Lane E).
- R05/8 audit: PollGroupEngine.Unsubscribe has no production caller; every
  driver already uses UnsubscribeAsync (no change).

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:36:12 -04:00
parent 6e2febbf3d
commit 79d6b0a4b2
11 changed files with 460 additions and 26 deletions
@@ -27,7 +27,27 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
/// </remarks>
public sealed class TelemetryLocalHub : ITelemetryLocalHub
{
/// <summary>
/// How long an eviction tombstone (see <see cref="_evicted"/>) is kept. It only has to outlive the
/// removed child's last late write — an in-flight capability call that completes after
/// <c>StopChild</c>, bounded by the longest capability timeout (seconds) plus the child's own stop —
/// so ten minutes is a wide margin. Without a sweep a driver removed for good kept its id forever.
/// </summary>
public static readonly TimeSpan EvictionTombstoneTtl = TimeSpan.FromMinutes(10);
private readonly object _gate = new();
private readonly TimeProvider _time;
/// <summary>Initializes a hub on the system clock.</summary>
public TelemetryLocalHub() : this(TimeProvider.System) { }
/// <summary>Initializes a hub on <paramref name="time"/> — the tombstone-sweep test seam.</summary>
/// <param name="time">The clock the eviction tombstones are stamped and swept by.</param>
public TelemetryLocalHub(TimeProvider time)
{
ArgumentNullException.ThrowIfNull(time);
_time = time;
}
// Plain Dictionary, NOT ConcurrentDictionary: every access is already under _gate, and
// ConcurrentDictionary.Values would materialize a fresh List (and take its own internal lock) on
@@ -42,9 +62,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
private readonly ConcurrentDictionary<string, TelemetryItem.Health> _healthCache = new();
private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
// Instances evicted and not since readmitted. Their late Health/Resilience items are dropped, so a
// removed driver's still-stopping child cannot re-prime the cache it was just evicted from. Under _gate.
private readonly HashSet<string> _evicted = new(StringComparer.Ordinal);
// Instances evicted and not since readmitted, stamped with the eviction time. Their late Health/Resilience
// items are dropped, so a removed driver's still-stopping child cannot re-prime the cache it was just
// evicted from. Swept past EvictionTombstoneTtl on the next Evict (never on the Emit hot path). Under _gate.
private readonly Dictionary<string, DateTimeOffset> _evicted = new(StringComparer.Ordinal);
/// <inheritdoc />
public void Emit(TelemetryItem item)
@@ -55,9 +76,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
{
switch (item)
{
case TelemetryItem.Health h when _evicted.Contains(h.E.DriverInstanceId):
case TelemetryItem.Health h when _evicted.ContainsKey(h.E.DriverInstanceId):
return;
case TelemetryItem.Resilience r when _evicted.Contains(r.E.DriverInstanceId):
case TelemetryItem.Resilience r when _evicted.ContainsKey(r.E.DriverInstanceId):
return;
case TelemetryItem.Health h:
_healthCache[h.E.DriverInstanceId] = h;
@@ -80,7 +101,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
lock (_gate)
{
_evicted.Add(driverInstanceId);
var now = _time.GetUtcNow();
SweepExpiredTombstones(now);
_evicted[driverInstanceId] = now;
_healthCache.TryRemove(driverInstanceId, out _);
foreach (var key in _resilienceCache.Keys)
if (string.Equals(key.InstanceId, driverInstanceId, StringComparison.Ordinal))
@@ -97,6 +120,24 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
_evicted.Remove(driverInstanceId);
}
/// <summary>Drops tombstones older than <see cref="EvictionTombstoneTtl"/>. Called under <see cref="_gate"/>
/// from <see cref="Evict"/> only — the set is bounded by distinct removed ids, so a linear pass per
/// eviction is cheap, and Emit never pays for it.</summary>
private void SweepExpiredTombstones(DateTimeOffset now)
{
if (_evicted.Count == 0) return;
List<string>? expired = null;
foreach (var (id, evictedAt) in _evicted)
{
if (now - evictedAt >= EvictionTombstoneTtl)
(expired ??= new List<string>()).Add(id);
}
if (expired is null) return;
foreach (var id in expired)
_evicted.Remove(id);
}
/// <inheritdoc />
public ITelemetrySubscription Subscribe(int boundedCapacity)
{