fix(runtime): lifecycle housekeeping — reprovision stamp, Stale drops, off-thread DB probe, tombstone sweep (A4)
- DriverHostActor.StopChild removes the driver's _lastReprovisionUtc stamp (and its deferred-reprovision timer): the map grew per removed id and a re-added driver inherited a backoff it never earned (02/L2). - Stale now registers explicit drops for AttributeValueBatchPublished, AttributeAlarmPublished, ArtifactForApply and ApplyWedged instead of letting them hit Unhandled (02/H5). - DbHealthProbeActor runs its SELECT on the thread pool and pipes the outcome back (single-flight), with a 2 s command timeout on relational providers, so a hung SQL Server no longer pins the actor thread and the cached GetStatus answer stays reachable (02/P7). - TelemetryLocalHub eviction tombstones are timestamped and swept past a 10 min TTL on the next Evict; a TimeProvider ctor is the test seam (02/L4). - Tests: ValuePublishes.Single helper replaces the per-value AttributeValuePublished in CalculationDependencyFlowTests (02/U4, test half; the production receive is owned by Lane E). - R05/8 audit: PollGroupEngine.Unsubscribe has no production caller; every driver already uses UnsubscribeAsync (no change). Refs #731 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -7,21 +7,37 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Single-flight cached health probe against the ConfigDb. Reads cached state via
|
||||
/// <c>Ask<DbHealthStatus></c>; a single SELECT 1 runs at most every <c>RefreshInterval</c>.
|
||||
/// <c>Ask<DbHealthStatus></c>; a single SELECT runs at most every <c>RefreshInterval</c>.
|
||||
/// Consumed by both the host's <c>/health/ready</c> endpoint and
|
||||
/// <c>RedundancyStateActor</c>'s stale calc.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The probe runs OFF the mailbox (a thread-pool task that pipes its outcome back), with a bounded
|
||||
/// command timeout. It used to run synchronously on the actor thread, relying on SqlClient's implicit
|
||||
/// 15 s connect / 30 s command timeouts — so a hanging SQL Server pinned a dispatcher thread for up to
|
||||
/// 30 s per tick, later ticks queued behind it, and the cached <see cref="GetStatus"/> answer was
|
||||
/// unreachable for exactly as long as the database was. Single-flight: a tick that lands while a probe
|
||||
/// is still running is skipped rather than stacked.
|
||||
/// </remarks>
|
||||
public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Bound on the probe's SQL command, relational providers only (the in-memory provider the
|
||||
/// actor tests run against has no command timeout).</summary>
|
||||
public static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(2);
|
||||
|
||||
public sealed class GetStatus { public static readonly GetStatus Instance = new(); private GetStatus() { } }
|
||||
public sealed record DbHealthStatus(bool Reachable, DateTime AsOfUtc, string? LastError);
|
||||
public sealed class Tick { public static readonly Tick Instance = new(); private Tick() { } }
|
||||
|
||||
/// <summary>Piped back by the off-thread probe with its outcome.</summary>
|
||||
private sealed record ProbeCompleted(DbHealthStatus Status);
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
private DbHealthStatus _last = new(false, DateTime.MinValue, "not probed yet");
|
||||
private bool _probeInFlight;
|
||||
|
||||
/// <summary>Gets or sets the timer scheduler for periodic health probes.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
@@ -38,28 +54,54 @@ public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
Receive<GetStatus>(_ => Sender.Tell(_last));
|
||||
Receive<Tick>(_ => RunProbe());
|
||||
Receive<Tick>(_ => StartProbe());
|
||||
Receive<ProbeCompleted>(msg =>
|
||||
{
|
||||
_probeInFlight = false;
|
||||
_last = msg.Status;
|
||||
if (!msg.Status.Reachable)
|
||||
_log.Warning("DbHealthProbe: probe failed: {Error}", msg.Status.LastError);
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
RunProbe();
|
||||
StartProbe();
|
||||
Timers.StartPeriodicTimer("probe", Tick.Instance, RefreshInterval);
|
||||
}
|
||||
|
||||
private void RunProbe()
|
||||
private void StartProbe()
|
||||
{
|
||||
if (_probeInFlight)
|
||||
{
|
||||
_log.Debug("DbHealthProbe: previous probe still running; skipping this tick");
|
||||
return;
|
||||
}
|
||||
|
||||
_probeInFlight = true;
|
||||
var factory = _dbFactory;
|
||||
// Never touches actor state; the outcome comes back as a message.
|
||||
Task.Run(() => Probe(factory)).PipeTo(
|
||||
Self,
|
||||
success: status => new ProbeCompleted(status),
|
||||
failure: ex => new ProbeCompleted(new DbHealthStatus(false, DateTime.UtcNow, ex.Message)));
|
||||
}
|
||||
|
||||
private static DbHealthStatus Probe(IDbContextFactory<OtOpcUaConfigDbContext> factory)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var db = _dbFactory.CreateDbContext();
|
||||
using var db = factory.CreateDbContext();
|
||||
// Local to this lease — the pool resets it on return (see DriverHostActor.LoadArtifactBlob).
|
||||
if (db.Database.IsRelational())
|
||||
db.Database.SetCommandTimeout(CommandTimeout);
|
||||
_ = db.Deployments.AsNoTracking().Take(1).ToList();
|
||||
_last = new DbHealthStatus(true, DateTime.UtcNow, null);
|
||||
return new DbHealthStatus(true, DateTime.UtcNow, null);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_last = new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
|
||||
_log.Warning(ex, "DbHealthProbe: probe failed");
|
||||
return new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user