Joseph Doherty
826312befe
Clears CommentChecker findings in src/ (3,280 -> 209, the remainder being false positives like PLC-5/IEEE-754/M0.0) by adding missing summaries/params/returns, normalising inheritdoc usage, and removing Gitea/review tracking IDs while keeping the rationale they annotated. Comment-only: comment-stripped sources are identical to the parent commit.
268 lines
15 KiB
C#
268 lines
15 KiB
C#
using System.Collections.Concurrent;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|
|
|
/// <summary>
|
|
/// Process-singleton tracker of live resilience counters per
|
|
/// <c>(DriverInstanceId, HostName)</c>. Populated by the CapabilityInvoker (call
|
|
/// start/complete + success reset), the DriverResiliencePipelineBuilder's Polly
|
|
/// telemetry (retry failure, breaker open/close), and the MemoryTracking layer
|
|
/// (footprint). Read by <c>DriverResilienceStatusPublisherService</c> (Host, driver
|
|
/// nodes) which periodically publishes each snapshot to the
|
|
/// <c>driver-resilience-status</c> DistributedPubSub topic for the AdminUI
|
|
/// resilience panel.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Pure in-memory (no EF Core / SQL Server dependency) so tests can exercise it directly.
|
|
/// The operator-facing reader is the DPS publisher named above, mirroring the driver-health
|
|
/// flow — not the DB-table/HostedService shape originally sketched for Phase 6.1 Stream E
|
|
/// (the DPS reader was built instead).
|
|
/// </remarks>
|
|
public sealed class DriverResilienceStatusTracker
|
|
{
|
|
private readonly ConcurrentDictionary<StatusKey, ResilienceStatusSnapshot> _status = new();
|
|
|
|
// Instances retired by Retire (removed from the deployed plan) and not since Reinstated. Every
|
|
// Record* for one of these is ignored: the removed instance's child stops ASYNCHRONOUSLY, so a capability
|
|
// call it still had in flight completes through its invoker after the host's release and would otherwise
|
|
// re-create the entry — which the resilience publisher then re-emits every tick, forever. One entry per
|
|
// distinct removed id, so the set stays as small as the driver churn that fills it.
|
|
private readonly ConcurrentDictionary<string, byte> _retired = new(StringComparer.Ordinal);
|
|
|
|
/// <summary>
|
|
/// The single write path for every <c>Record*</c>. A write for a retired instance is dropped; the
|
|
/// post-write re-check closes the race with a concurrent <see cref="Retire"/> — a writer that passed the
|
|
/// first check just before the retire and lands its upsert just after the retire's sweep removes its own
|
|
/// write again, so no interleaving resurrects the instance.
|
|
/// </summary>
|
|
private void Upsert(
|
|
string driverInstanceId,
|
|
string hostName,
|
|
Func<StatusKey, ResilienceStatusSnapshot> add,
|
|
Func<StatusKey, ResilienceStatusSnapshot, ResilienceStatusSnapshot> update)
|
|
{
|
|
if (_retired.ContainsKey(driverInstanceId)) return;
|
|
var key = new StatusKey(driverInstanceId, hostName);
|
|
_status.AddOrUpdate(key, add, update);
|
|
if (_retired.ContainsKey(driverInstanceId)) _status.TryRemove(key, out _);
|
|
}
|
|
|
|
/// <summary>Record a Polly pipeline failure for <paramref name="hostName"/>.</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <param name="utcNow">The UTC timestamp of the failure event.</param>
|
|
public void RecordFailure(string driverInstanceId, string hostName, DateTime utcNow)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { ConsecutiveFailures = 1, LastSampledUtc = utcNow },
|
|
(_, existing) => existing with
|
|
{
|
|
ConsecutiveFailures = existing.ConsecutiveFailures + 1,
|
|
LastSampledUtc = utcNow,
|
|
});
|
|
}
|
|
|
|
/// <summary>Reset the consecutive-failure count on a successful pipeline execution.</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <param name="utcNow">The UTC timestamp of the success event.</param>
|
|
public void RecordSuccess(string driverInstanceId, string hostName, DateTime utcNow)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { ConsecutiveFailures = 0, LastSampledUtc = utcNow },
|
|
(_, existing) => existing with
|
|
{
|
|
ConsecutiveFailures = 0,
|
|
LastSampledUtc = utcNow,
|
|
});
|
|
}
|
|
|
|
/// <summary>Record a circuit-breaker open event.</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <param name="utcNow">The UTC timestamp of the breaker open event.</param>
|
|
public void RecordBreakerOpen(string driverInstanceId, string hostName, DateTime utcNow)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow },
|
|
(_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow });
|
|
}
|
|
|
|
/// <summary>
|
|
/// Record a circuit-breaker close (recovery) event — the paired counterpart to
|
|
/// <see cref="RecordBreakerOpen"/>. Stamps <see cref="ResilienceStatusSnapshot.LastBreakerClosedUtc"/>
|
|
/// so <see cref="ResilienceStatusSnapshot.IsBreakerOpen"/> can report whether the breaker
|
|
/// is currently open. Called from the pipeline builder's <c>OnClosed</c> hook.
|
|
/// </summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <param name="utcNow">The UTC timestamp of the breaker close event.</param>
|
|
public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow },
|
|
(_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow });
|
|
}
|
|
|
|
/// <summary>
|
|
/// Drop every snapshot for one driver instance. Called by
|
|
/// <c>DriverCapabilityInvokerFactory.Create</c> alongside
|
|
/// <c>DriverResiliencePipelineBuilder.Invalidate</c>: the pipelines carry the actual breaker
|
|
/// state, so once they are discarded on a respawn, a retained snapshot describes a breaker
|
|
/// that no longer exists — and a stale "Breaker OPEN" could never self-correct, because only
|
|
/// the discarded pipeline's <c>OnClosed</c> hook stamps the close.
|
|
/// </summary>
|
|
/// <param name="driverInstanceId">The driver instance whose snapshots should be dropped.</param>
|
|
/// <returns>The number of snapshots removed.</returns>
|
|
public int Invalidate(string driverInstanceId)
|
|
{
|
|
var removed = 0;
|
|
foreach (var key in _status.Keys)
|
|
{
|
|
if (key.DriverInstanceId == driverInstanceId && _status.TryRemove(key, out _))
|
|
removed++;
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Drop every snapshot for a driver instance that has been REMOVED from the deployed plan, and ignore
|
|
/// every later write for it until <see cref="Reinstate"/>. Called by
|
|
/// <c>DriverCapabilityInvokerFactory.Release</c>. Unlike <see cref="Invalidate"/> (a respawn, whose new
|
|
/// invoker records straight away), the removed instance's child is still stopping — a call it had in
|
|
/// flight finishes after this returns, and without the hold its RecordCallStart / RecordSuccess /
|
|
/// RecordCallComplete re-create the entry the publisher then re-emits forever.
|
|
/// </summary>
|
|
/// <param name="driverInstanceId">The removed driver instance.</param>
|
|
/// <returns>The number of snapshots removed.</returns>
|
|
public int Retire(string driverInstanceId)
|
|
{
|
|
// Retire BEFORE sweeping: a write racing the sweep either lands first (and is swept) or re-checks after
|
|
// its upsert and removes itself (see Upsert).
|
|
_retired[driverInstanceId] = 0;
|
|
return Invalidate(driverInstanceId);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lift an earlier <see cref="Retire"/> so the instance records again — the same driver instance id
|
|
/// re-added in a later deployment. Called by <c>DriverCapabilityInvokerFactory.Create</c>; a no-op for
|
|
/// an id that was never retired.
|
|
/// </summary>
|
|
/// <param name="driverInstanceId">The driver instance being (re)created.</param>
|
|
public void Reinstate(string driverInstanceId) => _retired.TryRemove(driverInstanceId, out _);
|
|
|
|
/// <summary>Record a process recycle event (Tier C only).</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <param name="utcNow">The UTC timestamp of the recycle event.</param>
|
|
public void RecordRecycle(string driverInstanceId, string hostName, DateTime utcNow)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { LastRecycleUtc = utcNow, LastSampledUtc = utcNow },
|
|
(_, existing) => existing with { LastRecycleUtc = utcNow, LastSampledUtc = utcNow });
|
|
}
|
|
|
|
/// <summary>Capture / update the MemoryTracking-supplied baseline + current footprint.</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <param name="baselineBytes">The baseline footprint in bytes.</param>
|
|
/// <param name="currentBytes">The current footprint in bytes.</param>
|
|
/// <param name="utcNow">The UTC timestamp of the footprint update.</param>
|
|
public void RecordFootprint(string driverInstanceId, string hostName, long baselineBytes, long currentBytes, DateTime utcNow)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot
|
|
{
|
|
BaselineFootprintBytes = baselineBytes,
|
|
CurrentFootprintBytes = currentBytes,
|
|
LastSampledUtc = utcNow,
|
|
},
|
|
(_, existing) => existing with
|
|
{
|
|
BaselineFootprintBytes = baselineBytes,
|
|
CurrentFootprintBytes = currentBytes,
|
|
LastSampledUtc = utcNow,
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// Record the entry of a capability call for this (instance, host). Increments the
|
|
/// in-flight counter used as the <see cref="ResilienceStatusSnapshot.CurrentInFlight"/>
|
|
/// surface (in-flight capability-call depth). Paired with
|
|
/// <see cref="RecordCallComplete"/>; callers use try/finally.
|
|
/// </summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
public void RecordCallStart(string driverInstanceId, string hostName)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { CurrentInFlight = 1 },
|
|
(_, existing) => existing with { CurrentInFlight = existing.CurrentInFlight + 1 });
|
|
}
|
|
|
|
/// <summary>Paired with <see cref="RecordCallStart"/> — decrements the in-flight counter.</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
public void RecordCallComplete(string driverInstanceId, string hostName)
|
|
{
|
|
Upsert(driverInstanceId, hostName,
|
|
_ => new ResilienceStatusSnapshot { CurrentInFlight = 0 }, // start-without-complete shouldn't happen; clamp to 0
|
|
(_, existing) => existing with { CurrentInFlight = Math.Max(0, existing.CurrentInFlight - 1) });
|
|
}
|
|
|
|
/// <summary>Snapshot of a specific (instance, host) pair; null if no counters recorded yet.</summary>
|
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
|
/// <param name="hostName">The host name.</param>
|
|
/// <returns>The current resilience status snapshot, or null if no counters have been recorded for this pair.</returns>
|
|
public ResilienceStatusSnapshot? TryGet(string driverInstanceId, string hostName) =>
|
|
_status.TryGetValue(new StatusKey(driverInstanceId, hostName), out var snapshot) ? snapshot : null;
|
|
|
|
/// <summary>Copy of every currently-tracked (instance, host, snapshot) triple. Safe under concurrent writes.</summary>
|
|
/// <returns>A snapshot list of all tracked (driver instance, host, resilience snapshot) triples.</returns>
|
|
public IReadOnlyList<(string DriverInstanceId, string HostName, ResilienceStatusSnapshot Snapshot)> Snapshot() =>
|
|
_status.Select(kvp => (kvp.Key.DriverInstanceId, kvp.Key.HostName, kvp.Value)).ToList();
|
|
|
|
private readonly record struct StatusKey(string DriverInstanceId, string HostName);
|
|
}
|
|
|
|
/// <summary>Snapshot of the resilience counters for one <c>(DriverInstanceId, HostName)</c> pair.</summary>
|
|
public sealed record ResilienceStatusSnapshot
|
|
{
|
|
/// <summary>Gets the number of consecutive pipeline failures.</summary>
|
|
public int ConsecutiveFailures { get; init; }
|
|
/// <summary>Gets the UTC timestamp of the last circuit-breaker open event, if any.</summary>
|
|
public DateTime? LastBreakerOpenUtc { get; init; }
|
|
/// <summary>Gets the UTC timestamp of the last circuit-breaker close (recovery) event, if any.</summary>
|
|
public DateTime? LastBreakerClosedUtc { get; init; }
|
|
/// <summary>Gets the UTC timestamp of the last recycle event, if any.</summary>
|
|
public DateTime? LastRecycleUtc { get; init; }
|
|
|
|
/// <summary>
|
|
/// Gets whether the circuit breaker is currently open for this (instance, host):
|
|
/// an open event has been recorded and no later close event has superseded it.
|
|
/// Note: Polly half-opens after <c>BreakDuration</c> without firing <c>OnClosed</c>
|
|
/// (which fires only on a successful probe), so an idle driver can read "open" until
|
|
/// the next call self-corrects it.
|
|
/// </summary>
|
|
public bool IsBreakerOpen =>
|
|
LastBreakerOpenUtc is { } open &&
|
|
(LastBreakerClosedUtc is not { } closed || open > closed);
|
|
/// <summary>Gets the baseline process footprint in bytes.</summary>
|
|
public long BaselineFootprintBytes { get; init; }
|
|
/// <summary>Gets the current process footprint in bytes.</summary>
|
|
public long CurrentFootprintBytes { get; init; }
|
|
/// <summary>Gets the UTC timestamp when the counters were last updated.</summary>
|
|
public DateTime LastSampledUtc { get; init; }
|
|
|
|
/// <summary>
|
|
/// In-flight capability calls against this (instance, host). Bumped on call entry +
|
|
/// decremented on completion. Surfaced to the AdminUI's driver status panel as
|
|
/// <c>DriverResilienceStatusChanged.CurrentInFlight</c> over the
|
|
/// <c>driver-resilience-status</c> telemetry channel.
|
|
/// <para>It used to say this fed <c>DriverInstanceResilienceStatus.CurrentBulkheadDepth</c> "until
|
|
/// the full telemetry observer lands". The telemetry observer landed (Phase 5); that table never
|
|
/// had a writer and was since dropped.</para>
|
|
/// </summary>
|
|
public int CurrentInFlight { get; init; }
|
|
}
|