using System.Collections.Concurrent; namespace ZB.MOM.WW.OtOpcUa.Core.Resilience; /// /// Process-singleton tracker of live resilience counters per /// (DriverInstanceId, HostName). Populated by the CapabilityInvoker (call /// start/complete + success reset), the DriverResiliencePipelineBuilder's Polly /// telemetry (retry failure, breaker open/close), and the MemoryTracking layer /// (footprint). Read by DriverResilienceStatusPublisherService (Host, driver /// nodes) which periodically publishes each snapshot to the /// driver-resilience-status DistributedPubSub topic for the AdminUI /// resilience panel. /// /// /// Pure in-memory (no EF Core / SQL Server dependency) so tests can exercise it directly. /// The operator-facing reader is the DPS publisher named above, mirroring the driver-health /// flow — not the DB-table/HostedService shape originally sketched for Phase 6.1 Stream E /// (the DPS reader was built instead). /// public sealed class DriverResilienceStatusTracker { private readonly ConcurrentDictionary _status = new(); // Instances retired by Retire (removed from the deployed plan) and not since Reinstated. Every // Record* for one of these is ignored: the removed instance's child stops ASYNCHRONOUSLY, so a capability // call it still had in flight completes through its invoker after the host's release and would otherwise // re-create the entry — which the resilience publisher then re-emits every tick, forever. One entry per // distinct removed id, so the set stays as small as the driver churn that fills it. private readonly ConcurrentDictionary _retired = new(StringComparer.Ordinal); /// /// The single write path for every Record*. A write for a retired instance is dropped; the /// post-write re-check closes the race with a concurrent — a writer that passed the /// first check just before the retire and lands its upsert just after the retire's sweep removes its own /// write again, so no interleaving resurrects the instance. /// private void Upsert( string driverInstanceId, string hostName, Func add, Func update) { if (_retired.ContainsKey(driverInstanceId)) return; var key = new StatusKey(driverInstanceId, hostName); _status.AddOrUpdate(key, add, update); if (_retired.ContainsKey(driverInstanceId)) _status.TryRemove(key, out _); } /// Record a Polly pipeline failure for . /// The driver instance identifier. /// The host name. /// The UTC timestamp of the failure event. public void RecordFailure(string driverInstanceId, string hostName, DateTime utcNow) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { ConsecutiveFailures = 1, LastSampledUtc = utcNow }, (_, existing) => existing with { ConsecutiveFailures = existing.ConsecutiveFailures + 1, LastSampledUtc = utcNow, }); } /// Reset the consecutive-failure count on a successful pipeline execution. /// The driver instance identifier. /// The host name. /// The UTC timestamp of the success event. public void RecordSuccess(string driverInstanceId, string hostName, DateTime utcNow) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { ConsecutiveFailures = 0, LastSampledUtc = utcNow }, (_, existing) => existing with { ConsecutiveFailures = 0, LastSampledUtc = utcNow, }); } /// Record a circuit-breaker open event. /// The driver instance identifier. /// The host name. /// The UTC timestamp of the breaker open event. public void RecordBreakerOpen(string driverInstanceId, string hostName, DateTime utcNow) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow }, (_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow }); } /// /// Record a circuit-breaker close (recovery) event — the paired counterpart to /// . Stamps /// so can report whether the breaker /// is currently open. Called from the pipeline builder's OnClosed hook. /// /// The driver instance identifier. /// The host name. /// The UTC timestamp of the breaker close event. public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }, (_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }); } /// /// Drop every snapshot for one driver instance. Called by /// DriverCapabilityInvokerFactory.Create alongside /// DriverResiliencePipelineBuilder.Invalidate: the pipelines carry the actual breaker /// state, so once they are discarded on a respawn, a retained snapshot describes a breaker /// that no longer exists — and a stale "Breaker OPEN" could never self-correct, because only /// the discarded pipeline's OnClosed hook stamps the close. /// /// The driver instance whose snapshots should be dropped. /// The number of snapshots removed. public int Invalidate(string driverInstanceId) { var removed = 0; foreach (var key in _status.Keys) { if (key.DriverInstanceId == driverInstanceId && _status.TryRemove(key, out _)) removed++; } return removed; } /// /// Drop every snapshot for a driver instance that has been REMOVED from the deployed plan, and ignore /// every later write for it until . Called by /// DriverCapabilityInvokerFactory.Release. Unlike (a respawn, whose new /// invoker records straight away), the removed instance's child is still stopping — a call it had in /// flight finishes after this returns, and without the hold its RecordCallStart / RecordSuccess / /// RecordCallComplete re-create the entry the publisher then re-emits forever. /// /// The removed driver instance. /// The number of snapshots removed. public int Retire(string driverInstanceId) { // Retire BEFORE sweeping: a write racing the sweep either lands first (and is swept) or re-checks after // its upsert and removes itself (see Upsert). _retired[driverInstanceId] = 0; return Invalidate(driverInstanceId); } /// /// Lift an earlier so the instance records again — the same driver instance id /// re-added in a later deployment. Called by DriverCapabilityInvokerFactory.Create; a no-op for /// an id that was never retired. /// /// The driver instance being (re)created. public void Reinstate(string driverInstanceId) => _retired.TryRemove(driverInstanceId, out _); /// Record a process recycle event (Tier C only). /// The driver instance identifier. /// The host name. /// The UTC timestamp of the recycle event. public void RecordRecycle(string driverInstanceId, string hostName, DateTime utcNow) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { LastRecycleUtc = utcNow, LastSampledUtc = utcNow }, (_, existing) => existing with { LastRecycleUtc = utcNow, LastSampledUtc = utcNow }); } /// Capture / update the MemoryTracking-supplied baseline + current footprint. /// The driver instance identifier. /// The host name. /// The baseline footprint in bytes. /// The current footprint in bytes. /// The UTC timestamp of the footprint update. public void RecordFootprint(string driverInstanceId, string hostName, long baselineBytes, long currentBytes, DateTime utcNow) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { BaselineFootprintBytes = baselineBytes, CurrentFootprintBytes = currentBytes, LastSampledUtc = utcNow, }, (_, existing) => existing with { BaselineFootprintBytes = baselineBytes, CurrentFootprintBytes = currentBytes, LastSampledUtc = utcNow, }); } /// /// Record the entry of a capability call for this (instance, host). Increments the /// in-flight counter used as the /// surface (in-flight capability-call depth). Paired with /// ; callers use try/finally. /// /// The driver instance identifier. /// The host name. public void RecordCallStart(string driverInstanceId, string hostName) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { CurrentInFlight = 1 }, (_, existing) => existing with { CurrentInFlight = existing.CurrentInFlight + 1 }); } /// Paired with — decrements the in-flight counter. /// The driver instance identifier. /// The host name. public void RecordCallComplete(string driverInstanceId, string hostName) { Upsert(driverInstanceId, hostName, _ => new ResilienceStatusSnapshot { CurrentInFlight = 0 }, // start-without-complete shouldn't happen; clamp to 0 (_, existing) => existing with { CurrentInFlight = Math.Max(0, existing.CurrentInFlight - 1) }); } /// Snapshot of a specific (instance, host) pair; null if no counters recorded yet. /// The driver instance identifier. /// The host name. /// The current resilience status snapshot, or null if no counters have been recorded for this pair. public ResilienceStatusSnapshot? TryGet(string driverInstanceId, string hostName) => _status.TryGetValue(new StatusKey(driverInstanceId, hostName), out var snapshot) ? snapshot : null; /// Copy of every currently-tracked (instance, host, snapshot) triple. Safe under concurrent writes. /// A snapshot list of all tracked (driver instance, host, resilience snapshot) triples. public IReadOnlyList<(string DriverInstanceId, string HostName, ResilienceStatusSnapshot Snapshot)> Snapshot() => _status.Select(kvp => (kvp.Key.DriverInstanceId, kvp.Key.HostName, kvp.Value)).ToList(); private readonly record struct StatusKey(string DriverInstanceId, string HostName); } /// Snapshot of the resilience counters for one (DriverInstanceId, HostName) pair. public sealed record ResilienceStatusSnapshot { /// Gets the number of consecutive pipeline failures. public int ConsecutiveFailures { get; init; } /// Gets the UTC timestamp of the last circuit-breaker open event, if any. public DateTime? LastBreakerOpenUtc { get; init; } /// Gets the UTC timestamp of the last circuit-breaker close (recovery) event, if any. public DateTime? LastBreakerClosedUtc { get; init; } /// Gets the UTC timestamp of the last recycle event, if any. public DateTime? LastRecycleUtc { get; init; } /// /// Gets whether the circuit breaker is currently open for this (instance, host): /// an open event has been recorded and no later close event has superseded it. /// Note: Polly half-opens after BreakDuration without firing OnClosed /// (which fires only on a successful probe), so an idle driver can read "open" until /// the next call self-corrects it. /// public bool IsBreakerOpen => LastBreakerOpenUtc is { } open && (LastBreakerClosedUtc is not { } closed || open > closed); /// Gets the baseline process footprint in bytes. public long BaselineFootprintBytes { get; init; } /// Gets the current process footprint in bytes. public long CurrentFootprintBytes { get; init; } /// Gets the UTC timestamp when the counters were last updated. public DateTime LastSampledUtc { get; init; } /// /// In-flight capability calls against this (instance, host). Bumped on call entry + /// decremented on completion. Surfaced to the AdminUI's driver status panel as /// DriverResilienceStatusChanged.CurrentInFlight over the /// driver-resilience-status telemetry channel. /// It used to say this fed DriverInstanceResilienceStatus.CurrentBulkheadDepth "until /// the full telemetry observer lands". The telemetry observer landed (Phase 5); that table never /// had a writer and was since dropped. /// public int CurrentInFlight { get; init; } }