using System.Collections.Concurrent;
namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
///
/// Process-singleton tracker of live resilience counters per
/// (DriverInstanceId, HostName). Populated by the CapabilityInvoker (call
/// start/complete + success reset), the DriverResiliencePipelineBuilder's Polly
/// telemetry (retry failure, breaker open/close), and the MemoryTracking layer
/// (footprint). Read by DriverResilienceStatusPublisherService (Host, driver
/// nodes) which periodically publishes each snapshot to the
/// driver-resilience-status DistributedPubSub topic for the AdminUI
/// resilience panel.
///
///
/// Pure in-memory (no EF Core / SQL Server dependency) so tests can exercise it directly.
/// The operator-facing reader is the DPS publisher named above, mirroring the driver-health
/// flow — not the DB-table/HostedService shape originally sketched for Phase 6.1 Stream E
/// (the DPS reader was built instead).
///
public sealed class DriverResilienceStatusTracker
{
private readonly ConcurrentDictionary _status = new();
// Instances retired by Retire (removed from the deployed plan) and not since Reinstated. Every
// Record* for one of these is ignored: the removed instance's child stops ASYNCHRONOUSLY, so a capability
// call it still had in flight completes through its invoker after the host's release and would otherwise
// re-create the entry — which the resilience publisher then re-emits every tick, forever. One entry per
// distinct removed id, so the set stays as small as the driver churn that fills it.
private readonly ConcurrentDictionary _retired = new(StringComparer.Ordinal);
///
/// The single write path for every Record*. A write for a retired instance is dropped; the
/// post-write re-check closes the race with a concurrent — a writer that passed the
/// first check just before the retire and lands its upsert just after the retire's sweep removes its own
/// write again, so no interleaving resurrects the instance.
///
private void Upsert(
string driverInstanceId,
string hostName,
Func add,
Func update)
{
if (_retired.ContainsKey(driverInstanceId)) return;
var key = new StatusKey(driverInstanceId, hostName);
_status.AddOrUpdate(key, add, update);
if (_retired.ContainsKey(driverInstanceId)) _status.TryRemove(key, out _);
}
/// Record a Polly pipeline failure for .
/// The driver instance identifier.
/// The host name.
/// The UTC timestamp of the failure event.
public void RecordFailure(string driverInstanceId, string hostName, DateTime utcNow)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { ConsecutiveFailures = 1, LastSampledUtc = utcNow },
(_, existing) => existing with
{
ConsecutiveFailures = existing.ConsecutiveFailures + 1,
LastSampledUtc = utcNow,
});
}
/// Reset the consecutive-failure count on a successful pipeline execution.
/// The driver instance identifier.
/// The host name.
/// The UTC timestamp of the success event.
public void RecordSuccess(string driverInstanceId, string hostName, DateTime utcNow)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { ConsecutiveFailures = 0, LastSampledUtc = utcNow },
(_, existing) => existing with
{
ConsecutiveFailures = 0,
LastSampledUtc = utcNow,
});
}
/// Record a circuit-breaker open event.
/// The driver instance identifier.
/// The host name.
/// The UTC timestamp of the breaker open event.
public void RecordBreakerOpen(string driverInstanceId, string hostName, DateTime utcNow)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow },
(_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow });
}
///
/// Record a circuit-breaker close (recovery) event — the paired counterpart to
/// . Stamps
/// so can report whether the breaker
/// is currently open. Called from the pipeline builder's OnClosed hook.
///
/// The driver instance identifier.
/// The host name.
/// The UTC timestamp of the breaker close event.
public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow },
(_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow });
}
///
/// Drop every snapshot for one driver instance. Called by
/// DriverCapabilityInvokerFactory.Create alongside
/// DriverResiliencePipelineBuilder.Invalidate: the pipelines carry the actual breaker
/// state, so once they are discarded on a respawn, a retained snapshot describes a breaker
/// that no longer exists — and a stale "Breaker OPEN" could never self-correct, because only
/// the discarded pipeline's OnClosed hook stamps the close.
///
/// The driver instance whose snapshots should be dropped.
/// The number of snapshots removed.
public int Invalidate(string driverInstanceId)
{
var removed = 0;
foreach (var key in _status.Keys)
{
if (key.DriverInstanceId == driverInstanceId && _status.TryRemove(key, out _))
removed++;
}
return removed;
}
///
/// Drop every snapshot for a driver instance that has been REMOVED from the deployed plan, and ignore
/// every later write for it until . Called by
/// DriverCapabilityInvokerFactory.Release. Unlike (a respawn, whose new
/// invoker records straight away), the removed instance's child is still stopping — a call it had in
/// flight finishes after this returns, and without the hold its RecordCallStart / RecordSuccess /
/// RecordCallComplete re-create the entry the publisher then re-emits forever.
///
/// The removed driver instance.
/// The number of snapshots removed.
public int Retire(string driverInstanceId)
{
// Retire BEFORE sweeping: a write racing the sweep either lands first (and is swept) or re-checks after
// its upsert and removes itself (see Upsert).
_retired[driverInstanceId] = 0;
return Invalidate(driverInstanceId);
}
///
/// Lift an earlier so the instance records again — the same driver instance id
/// re-added in a later deployment. Called by DriverCapabilityInvokerFactory.Create; a no-op for
/// an id that was never retired.
///
/// The driver instance being (re)created.
public void Reinstate(string driverInstanceId) => _retired.TryRemove(driverInstanceId, out _);
/// Record a process recycle event (Tier C only).
/// The driver instance identifier.
/// The host name.
/// The UTC timestamp of the recycle event.
public void RecordRecycle(string driverInstanceId, string hostName, DateTime utcNow)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { LastRecycleUtc = utcNow, LastSampledUtc = utcNow },
(_, existing) => existing with { LastRecycleUtc = utcNow, LastSampledUtc = utcNow });
}
/// Capture / update the MemoryTracking-supplied baseline + current footprint.
/// The driver instance identifier.
/// The host name.
/// The baseline footprint in bytes.
/// The current footprint in bytes.
/// The UTC timestamp of the footprint update.
public void RecordFootprint(string driverInstanceId, string hostName, long baselineBytes, long currentBytes, DateTime utcNow)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot
{
BaselineFootprintBytes = baselineBytes,
CurrentFootprintBytes = currentBytes,
LastSampledUtc = utcNow,
},
(_, existing) => existing with
{
BaselineFootprintBytes = baselineBytes,
CurrentFootprintBytes = currentBytes,
LastSampledUtc = utcNow,
});
}
///
/// Record the entry of a capability call for this (instance, host). Increments the
/// in-flight counter used as the
/// surface (in-flight capability-call depth). Paired with
/// ; callers use try/finally.
///
/// The driver instance identifier.
/// The host name.
public void RecordCallStart(string driverInstanceId, string hostName)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { CurrentInFlight = 1 },
(_, existing) => existing with { CurrentInFlight = existing.CurrentInFlight + 1 });
}
/// Paired with — decrements the in-flight counter.
/// The driver instance identifier.
/// The host name.
public void RecordCallComplete(string driverInstanceId, string hostName)
{
Upsert(driverInstanceId, hostName,
_ => new ResilienceStatusSnapshot { CurrentInFlight = 0 }, // start-without-complete shouldn't happen; clamp to 0
(_, existing) => existing with { CurrentInFlight = Math.Max(0, existing.CurrentInFlight - 1) });
}
/// Snapshot of a specific (instance, host) pair; null if no counters recorded yet.
/// The driver instance identifier.
/// The host name.
/// The current resilience status snapshot, or null if no counters have been recorded for this pair.
public ResilienceStatusSnapshot? TryGet(string driverInstanceId, string hostName) =>
_status.TryGetValue(new StatusKey(driverInstanceId, hostName), out var snapshot) ? snapshot : null;
/// Copy of every currently-tracked (instance, host, snapshot) triple. Safe under concurrent writes.
/// A snapshot list of all tracked (driver instance, host, resilience snapshot) triples.
public IReadOnlyList<(string DriverInstanceId, string HostName, ResilienceStatusSnapshot Snapshot)> Snapshot() =>
_status.Select(kvp => (kvp.Key.DriverInstanceId, kvp.Key.HostName, kvp.Value)).ToList();
private readonly record struct StatusKey(string DriverInstanceId, string HostName);
}
/// Snapshot of the resilience counters for one (DriverInstanceId, HostName) pair.
public sealed record ResilienceStatusSnapshot
{
/// Gets the number of consecutive pipeline failures.
public int ConsecutiveFailures { get; init; }
/// Gets the UTC timestamp of the last circuit-breaker open event, if any.
public DateTime? LastBreakerOpenUtc { get; init; }
/// Gets the UTC timestamp of the last circuit-breaker close (recovery) event, if any.
public DateTime? LastBreakerClosedUtc { get; init; }
/// Gets the UTC timestamp of the last recycle event, if any.
public DateTime? LastRecycleUtc { get; init; }
///
/// Gets whether the circuit breaker is currently open for this (instance, host):
/// an open event has been recorded and no later close event has superseded it.
/// Note: Polly half-opens after BreakDuration without firing OnClosed
/// (which fires only on a successful probe), so an idle driver can read "open" until
/// the next call self-corrects it.
///
public bool IsBreakerOpen =>
LastBreakerOpenUtc is { } open &&
(LastBreakerClosedUtc is not { } closed || open > closed);
/// Gets the baseline process footprint in bytes.
public long BaselineFootprintBytes { get; init; }
/// Gets the current process footprint in bytes.
public long CurrentFootprintBytes { get; init; }
/// Gets the UTC timestamp when the counters were last updated.
public DateTime LastSampledUtc { get; init; }
///
/// In-flight capability calls against this (instance, host). Bumped on call entry +
/// decremented on completion. Surfaced to the AdminUI's driver status panel as
/// DriverResilienceStatusChanged.CurrentInFlight over the
/// driver-resilience-status telemetry channel.
/// It used to say this fed DriverInstanceResilienceStatus.CurrentBulkheadDepth "until
/// the full telemetry observer lands". The telemetry observer landed (Phase 5); that table never
/// had a writer and was since dropped.
///
public int CurrentInFlight { get; init; }
}