fix(hosts): driver health is per (driver, node) — ReportingNodeId on DriverHealthChanged + telemetry proto (#847)

This commit is contained in:
Joseph Doherty
2026-11-11 03:41:59 -05:00
parent 3b1b32f0c1
commit 038aa007ac
11 changed files with 150 additions and 21 deletions
@@ -6,11 +6,17 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
/// <summary>
/// Thread-safe in-memory implementation of <see cref="IDriverStatusSnapshotStore"/>.
/// Keyed by <see cref="DriverHealthChanged.DriverInstanceId"/>; last write wins.
/// Keyed by (<see cref="DriverHealthChanged.DriverInstanceId"/>, <see cref="DriverHealthChanged.ReportingNodeId"/>);
/// last write wins per key.
/// </summary>
/// <remarks>
/// Keyed per reporting node since lmxopcua#847: both nodes of a redundant pair run the same driver instance,
/// so a driver-id-only key made a driver Faulted on central-1 and Healthy on central-2 flip between the two on
/// every publish. A snapshot from a node that predates <c>ReportingNodeId</c> keys under the empty node.
/// </remarks>
public sealed class InMemoryDriverStatusSnapshotStore : IDriverStatusSnapshotStore
{
private readonly ConcurrentDictionary<string, DriverHealthChanged> _byInstance = new();
private readonly ConcurrentDictionary<(string DriverInstanceId, string NodeId), DriverHealthChanged> _byInstance = new();
private readonly ILogger<InMemoryDriverStatusSnapshotStore>? _logger;
private readonly FailureLogThrottle _failureLog = new();
@@ -22,10 +28,12 @@ public sealed class InMemoryDriverStatusSnapshotStore : IDriverStatusSnapshotSto
/// <summary>Raised whenever a driver status snapshot is stored or updated.</summary>
public event Action<DriverHealthChanged>? SnapshotChanged;
private static (string, string) KeyOf(DriverHealthChanged s) => (s.DriverInstanceId, s.ReportingNodeId ?? "");
/// <inheritdoc />
public void Upsert(DriverHealthChanged snapshot)
{
_byInstance[snapshot.DriverInstanceId] = snapshot;
_byInstance[KeyOf(snapshot)] = snapshot;
// Capture-then-invoke so a concurrent unsubscribe can't null the delegate mid-raise; one
// subscriber at a time so a throwing one cannot starve the rest (or escape onto the bridge actor).
SafeMulticast.Invoke(SnapshotChanged, snapshot,
@@ -36,11 +44,11 @@ public sealed class InMemoryDriverStatusSnapshotStore : IDriverStatusSnapshotSto
public int EvictUnconfiguredStale(IReadOnlySet<string> configuredInstanceIds, DateTime olderThanUtc)
{
var removed = 0;
foreach (var (id, snapshot) in _byInstance)
foreach (var (key, snapshot) in _byInstance)
{
if (configuredInstanceIds.Contains(id) || snapshot.PublishedUtc >= olderThanUtc) continue;
if (configuredInstanceIds.Contains(key.DriverInstanceId) || snapshot.PublishedUtc >= olderThanUtc) continue;
// Remove only the exact snapshot inspected: a fresh publish that raced in is kept.
if (_byInstance.TryRemove(new KeyValuePair<string, DriverHealthChanged>(id, snapshot))) removed++;
if (_byInstance.TryRemove(new KeyValuePair<(string, string), DriverHealthChanged>(key, snapshot))) removed++;
}
return removed;
@@ -48,7 +56,17 @@ public sealed class InMemoryDriverStatusSnapshotStore : IDriverStatusSnapshotSto
/// <inheritdoc />
public bool TryGet(string driverInstanceId, out DriverHealthChanged snapshot)
=> _byInstance.TryGetValue(driverInstanceId, out snapshot!);
{
DriverHealthChanged? latest = null;
foreach (var (key, s) in _byInstance)
{
if (!string.Equals(key.DriverInstanceId, driverInstanceId, StringComparison.Ordinal)) continue;
if (latest is null || s.PublishedUtc > latest.PublishedUtc) latest = s;
}
snapshot = latest!;
return latest is not null;
}
/// <inheritdoc />
public IReadOnlyCollection<DriverHealthChanged> GetAll() => _byInstance.Values.ToArray();