feat(adminui): driver Controls status panel shows one entry per reporting node (#851)

DriverStatusPanel (the /hosts Controls dialog) showed one snapshot per driver - the most recently published across both nodes of the redundant pair - so a driver Faulted on one node and Healthy on its peer flipped on every publish. It now primes from the store's GetAll() and keeps one entry per ReportingNodeId (DriverStatusByNode): per-node state chip in the header, per-node last-success / 5-min errors / last error, per-node stale dimming. A snapshot with no node id (pre-#847 publisher) renders unlabelled, sorted last. A primed copy older than a racing live publish for the same node is dropped.
2026-11-11 06:16:06 -05:00
parent d1b6cad299
commit b40f08e187
3 changed files with 294 additions and 50 deletions
@@ -1,9 +1,11 @@
-@* Live driver-status panel — subscribes to /hubs/driverstatus and shows state chip,
-   last-success age, 5-min error count, and last error message.
+@* Live driver-status panel — reads the in-process driver-health snapshot store and shows, PER REPORTING NODE
+   (lmxopcua#851 — both nodes of a redundant pair run the driver), the state chip, last-success age, 5-min error
+   count and last error message. A snapshot from a node that predates ReportingNodeId (#847) renders unlabelled.
   Enabled=false renders a static "Disabled" notice and never opens the hub.
   DriverOperator-gated Reconnect/Restart buttons appear for authorised users. *@
@implements IAsyncDisposable
@using Microsoft.AspNetCore.Authorization
+@using ZB.MOM.WW.OtOpcUa.AdminUI.Hosts
@using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
@using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin
@@ -14,12 +16,18 @@
@inject IDriverStatusSnapshotStore StatusStore
@inject IDriverResilienceStatusStore ResilienceStore

-<section class="panel rise mt-3" style="animation-delay:.04s; @(_stale ? "opacity:0.5;" : "")">
-    <div class="panel-head d-flex align-items-center gap-2">
+@{ var nodes = _byNode?.Entries ?? []; var now = DateTime.UtcNow; }
+<section class="panel rise mt-3" style="animation-delay:.04s">
+    <div class="panel-head d-flex align-items-center gap-2 flex-wrap">
        <span>Driver status</span>
-        @if (_snapshot is not null)
+        @if (nodes.Count > 0)
        {
-            <span class="chip @ChipClass(_snapshot.State)">@_snapshot.State</span>
+            @foreach (var n in nodes)
+            {
+                <span @key="@($"head-{n.Key}")" class="chip @ChipClass(n.Snapshot.State)"
+                      style="@(n.IsStale(now, StaleAfter) ? "opacity:0.5;" : "")"
+                      title="@(n.NodeId is null ? "Reported by a node that predates per-node health (#847)" : $"Reported by {n.NodeId}")">@(nodes.Count > 1 || n.NodeId is not null ? $"{n.Label}: " : "")@n.Snapshot.State</span>
+            }
        }
        else if (!Enabled)
        {
@@ -40,37 +48,51 @@
        {
            <p class="mb-0" style="color:var(--bad)" title="@_error">SignalR error: @DisplayText.Abbreviate(_error, 60)</p>
        }
-        else if (_snapshot is null)
+        else if (nodes.Count == 0)
        {
            <p class="mb-0" style="color:var(--ink-faint)">Awaiting first snapshot&hellip;</p>
        }
        else
        {
-            <div class="d-flex flex-wrap gap-3 align-items-baseline">
-                <span style="color:var(--ink-soft)">
-                    Last success:
-                    @if (_snapshot.LastSuccessfulReadUtc is { } t)
-                    {
-                        <strong>@HumanizeAge(t) ago</strong>
-                    }
-                    else
-                    {
-                        <strong>never</strong>
-                    }
-                </span>
-
-                @if (_snapshot.ErrorCount5Min > 0)
-                {
-                    <span class="chip chip-bad">@_snapshot.ErrorCount5Min error@(_snapshot.ErrorCount5Min == 1 ? "" : "s") / 5 min</span>
-                }
-            </div>
-
-            @if (_snapshot.LastError is { Length: > 0 } lastError)
+            @foreach (var n in nodes)
            {
-                <details class="mt-2" style="font-size:0.85rem">
-                    <summary style="cursor:pointer; color:var(--ink-soft)">Last error</summary>
-                    <pre class="mt-1 mb-0" style="white-space:pre-wrap; word-break:break-word; color:var(--bad); font-size:0.8rem">@lastError</pre>
-                </details>
+                var snap = n.Snapshot;
+                <div @key="@($"node-{n.Key}")" class="@(n == nodes[0] ? "" : "mt-3 pt-2 border-top")"
+                     style="@(n.IsStale(now, StaleAfter) ? "opacity:0.5;" : "")">
+                    <div class="d-flex flex-wrap gap-3 align-items-baseline">
+                        <span class="mono" style="font-weight:600"
+                              title="@(n.NodeId is null ? "Reported by a node that predates per-node health (#847)" : "Reporting node")">@n.Label</span>
+                        <span class="chip @ChipClass(snap.State)">@snap.State</span>
+                        <span style="color:var(--ink-soft)">
+                            Last success:
+                            @if (snap.LastSuccessfulReadUtc is { } t)
+                            {
+                                <strong>@HumanizeAge(t) ago</strong>
+                            }
+                            else
+                            {
+                                <strong>never</strong>
+                            }
+                        </span>
+
+                        @if (snap.ErrorCount5Min > 0)
+                        {
+                            <span class="chip chip-bad">@snap.ErrorCount5Min error@(snap.ErrorCount5Min == 1 ? "" : "s") / 5 min</span>
+                        }
+                        @if (n.IsStale(now, StaleAfter))
+                        {
+                            <span class="chip chip-idle" title="No snapshot from this node for over @((int)StaleAfter.TotalSeconds) s">stale</span>
+                        }
+                    </div>
+
+                    @if (snap.LastError is { Length: > 0 } lastError)
+                    {
+                        <details class="mt-2" style="font-size:0.85rem">
+                            <summary style="cursor:pointer; color:var(--ink-soft)">Last error</summary>
+                            <pre class="mt-1 mb-0" style="white-space:pre-wrap; word-break:break-word; color:var(--bad); font-size:0.8rem">@lastError</pre>
+                        </details>
+                    }
+                </div>
            }
        }

@@ -194,9 +216,10 @@
    [Parameter] public string ClusterId { get; set; } = "";
    [Parameter] public bool Enabled { get; set; } = true;

-    private DriverHealthChanged? _snapshot;
-    private DateTime _lastUpdateUtc = DateTime.MinValue;
-    private bool _stale;
+    // One entry per reporting node (lmxopcua#851). Created in OnInitializedAsync once DriverInstanceId is bound.
+    // Staleness is per node: a node that stops publishing dims on its own while its peer stays live.
+    private DriverStatusByNode? _byNode;
+    private static readonly TimeSpan StaleAfter = TimeSpan.FromSeconds(30);

    // Per-host resilience snapshots for THIS instance, keyed by HostName. Fed by the
    // driver-resilience-status DPS store (read in-process, same house pattern as the health store).
@@ -234,13 +257,11 @@

        _connecting = true;

-        // Tick every 5 s to refresh the stale-dimming check and humanized ages.
-        _timer = new System.Threading.Timer(_ =>
-        {
-            _stale = _snapshot is not null &&
-                     (DateTime.UtcNow - _lastUpdateUtc).TotalSeconds > 30;
-            InvokeAsync(StateHasChanged);
-        }, null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
+        _byNode = new DriverStatusByNode(DriverInstanceId);
+
+        // Tick every 5 s to re-render the per-node stale dimming and humanized ages.
+        _timer = new System.Threading.Timer(_ => InvokeAsync(StateHasChanged),
+            null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));

        // Read live status straight from the in-process snapshot store rather than opening a
        // self-targeted SignalR connection. This component runs server-side (Blazor
@@ -252,12 +273,12 @@
        // this circuit.
        try
        {
+            // Subscribe BEFORE priming so a publish racing the read is not missed; a primed copy that lands after
+            // a newer racing live one is dropped by Upsert (it keeps the later PublishedUtc per node).
            StatusStore.SnapshotChanged += OnSnapshotChanged;
-            if (StatusStore.TryGet(DriverInstanceId, out var snap))
-            {
-                _snapshot = snap;
-                _lastUpdateUtc = DateTime.UtcNow;
-            }
+            var primedUtc = DateTime.UtcNow;
+            foreach (var snap in StatusStore.GetAll())
+                _byNode.Upsert(snap, primedUtc);

            // Prime + subscribe the resilience store (same in-process read pattern as the health store).
            ResilienceStore.SnapshotChanged += OnResilienceChanged;
@@ -278,12 +299,9 @@
    // ignore snapshots for other instances and marshal onto the render sync context.
    private void OnSnapshotChanged(DriverHealthChanged snap)
    {
-        if (!string.Equals(snap.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal))
+        if (_byNode is null || !_byNode.Upsert(snap, DateTime.UtcNow))
            return;

-        _snapshot = snap;
-        _lastUpdateUtc = DateTime.UtcNow;
-        _stale = false;
        InvokeAsync(StateHasChanged);
    }

@@ -0,0 +1,86 @@
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Hosts;
+
+/// <summary>
+///     One reporting node's last-known health for a driver, as the Controls dialog's
+///     <c>DriverStatusPanel</c> renders it (lmxopcua#851).
+/// </summary>
+/// <param name="NodeId">The reporting <c>ClusterNode.NodeId</c>; null when the snapshot came from a node that
+///     predates <see cref="DriverHealthChanged.ReportingNodeId"/> (#847) — it still renders, unlabelled.</param>
+/// <param name="Snapshot">The node's most recent snapshot.</param>
+/// <param name="ReceivedUtc">When THIS AdminUI node received it (local clock — drives the stale dimming, so a
+///     peer's clock skew cannot make a live row look stale).</param>
+public sealed record DriverNodeStatus(string? NodeId, DriverHealthChanged Snapshot, DateTime ReceivedUtc)
+{
+    /// <summary>Render key, unique per reporting node (the empty string stands for "no node id").</summary>
+    public string Key => NodeId ?? "";
+
+    /// <summary>The label the panel prints for the node.</summary>
+    public string Label => NodeId ?? "—";
+
+    /// <summary>True once no snapshot has arrived from this node for longer than <paramref name="staleAfter"/>.</summary>
+    public bool IsStale(DateTime nowUtc, TimeSpan staleAfter) => nowUtc - ReceivedUtc > staleAfter;
+}
+
+/// <summary>
+///     The per-node health view of ONE driver instance behind the Controls dialog's status panel (lmxopcua#851).
+///     Both nodes of a redundant pair run the same driver instance, so the panel shows one entry per reporting
+///     node instead of the single most-recently-published snapshot (which flipped between a Faulted node and a
+///     Healthy peer on every publish). Thread-safe: <see cref="Upsert"/> is called on the snapshot store's bridge
+///     thread while the render reads <see cref="Entries"/>.
+/// </summary>
+public sealed class DriverStatusByNode
+{
+    private readonly object _gate = new();
+    private readonly Dictionary<string, DriverNodeStatus> _byNode = new(StringComparer.Ordinal);
+
+    /// <summary>Initializes a new view for one driver instance.</summary>
+    /// <param name="driverInstanceId">The driver instance whose snapshots this view keeps; others are ignored.</param>
+    public DriverStatusByNode(string driverInstanceId) => DriverInstanceId = driverInstanceId;
+
+    /// <summary>The driver instance this view tracks.</summary>
+    public string DriverInstanceId { get; }
+
+    /// <summary>
+    ///     Records a snapshot as its node's latest. Snapshots for other driver instances are ignored, and so is one
+    ///     published EARLIER than the node's current entry (both stamped by the same node's clock): the panel primes
+    ///     from <c>GetAll()</c> after subscribing, so a live publish can land before the primed copy of an older
+    ///     one. Returns true when the view changed.
+    /// </summary>
+    /// <param name="snapshot">The published snapshot.</param>
+    /// <param name="receivedUtc">Local receive time.</param>
+    public bool Upsert(DriverHealthChanged snapshot, DateTime receivedUtc)
+    {
+        if (!string.Equals(snapshot.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal))
+            return false;
+
+        var nodeId = string.IsNullOrWhiteSpace(snapshot.ReportingNodeId) ? null : snapshot.ReportingNodeId;
+        var entry = new DriverNodeStatus(nodeId, snapshot, receivedUtc);
+        lock (_gate)
+        {
+            if (_byNode.TryGetValue(entry.Key, out var current) && snapshot.PublishedUtc < current.Snapshot.PublishedUtc)
+                return false;
+            _byNode[entry.Key] = entry;
+        }
+
+        return true;
+    }
+
+    /// <summary>
+    ///     A point-in-time copy of every node's entry, ordered by node id (ordinal, case-insensitive) with the
+    ///     unlabelled (pre-#847) entry last.
+    /// </summary>
+    public IReadOnlyList<DriverNodeStatus> Entries
+    {
+        get
+        {
+            DriverNodeStatus[] copy;
+            lock (_gate) copy = _byNode.Values.ToArray();
+            return copy
+                .OrderBy(e => e.NodeId is null ? 1 : 0)
+                .ThenBy(e => e.NodeId ?? "", StringComparer.OrdinalIgnoreCase)
+                .ToList();
+        }
+    }
+}
@@ -0,0 +1,140 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Hosts;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Hosts;
+
+/// <summary>
+///     Covers <see cref="DriverStatusByNode"/> — the per-node view behind the /hosts Controls dialog's
+///     <c>DriverStatusPanel</c> (lmxopcua#851). Before it the panel showed ONE snapshot per driver, the most
+///     recently published across both nodes of the redundant pair, so a driver Faulted on central-1 and Healthy on
+///     central-2 flipped between the two on every publish.
+/// </summary>
+[Trait("Category", "Unit")]
+public sealed class DriverStatusByNodeTests
+{
+    private static readonly DateTime T0 = new(2026, 11, 11, 12, 0, 0, DateTimeKind.Utc);
+
+    private static DriverHealthChanged Snap(string driver, string state, DateTime published, string? node, string? error = null) =>
+        new("MAIN", driver, state, null, error, error is null ? 0 : 1, published, ReportingNodeId: node);
+
+    [Fact]
+    public void Two_nodes_reporting_one_driver_yield_two_entries_ordered_by_node()
+    {
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("drv", "Healthy", T0, "central-2"), T0).ShouldBeTrue();
+        view.Upsert(Snap("drv", "Faulted", T0.AddSeconds(1), "central-1", "boom"), T0).ShouldBeTrue();
+
+        var entries = view.Entries;
+        entries.Select(e => e.NodeId).ShouldBe(["central-1", "central-2"]);
+        entries.Select(e => e.Snapshot.State).ShouldBe(["Faulted", "Healthy"]);
+        entries.Select(e => e.Key).Distinct().Count().ShouldBe(2);
+    }
+
+    [Fact]
+    public void A_later_publish_replaces_only_its_own_nodes_entry()
+    {
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("drv", "Faulted", T0, "central-1", "x"), T0);
+        view.Upsert(Snap("drv", "Healthy", T0, "central-2"), T0);
+        view.Upsert(Snap("drv", "Faulted", T0.AddSeconds(5), "central-1", "y"), T0.AddSeconds(5));
+
+        var entries = view.Entries;
+        entries.Count.ShouldBe(2);
+        entries.Single(e => e.NodeId == "central-1").Snapshot.LastError.ShouldBe("y");
+        entries.Single(e => e.NodeId == "central-2").Snapshot.State.ShouldBe("Healthy");
+    }
+
+    [Fact]
+    public void Snapshots_for_other_drivers_are_ignored()
+    {
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("other", "Faulted", T0, "central-1"), T0).ShouldBeFalse();
+        view.Entries.ShouldBeEmpty();
+    }
+
+    [Fact]
+    public void A_snapshot_from_a_pre_847_node_with_no_node_id_still_renders_unlabelled_and_sorts_last()
+    {
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("drv", "Healthy", T0, node: null), T0).ShouldBeTrue();
+        view.Upsert(Snap("drv", "Faulted", T0, "central-1"), T0);
+
+        var entries = view.Entries;
+        entries.Count.ShouldBe(2);
+        entries[0].NodeId.ShouldBe("central-1");
+        entries[1].NodeId.ShouldBeNull();
+        entries[1].Label.ShouldBe("—");
+        entries[1].Key.ShouldBe("");
+        entries[1].Snapshot.State.ShouldBe("Healthy");
+    }
+
+    [Fact]
+    public void A_blank_node_id_is_the_same_unlabelled_entry_as_a_missing_one()
+    {
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("drv", "Healthy", T0, node: null), T0);
+        view.Upsert(Snap("drv", "Faulted", T0.AddSeconds(1), node: "  "), T0);
+
+        view.Entries.ShouldHaveSingleItem().Snapshot.State.ShouldBe("Faulted");
+    }
+
+    [Fact]
+    public void An_older_publish_arriving_late_does_not_overwrite_a_newer_one_for_the_same_node()
+    {
+        // The panel subscribes, THEN primes from the store's GetAll(): a live publish can land before the primed
+        // copy of the previous snapshot. The primed (older) copy must not win.
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("drv", "Healthy", T0.AddSeconds(10), "central-1"), T0).ShouldBeTrue();
+        view.Upsert(Snap("drv", "Faulted", T0, "central-1"), T0).ShouldBeFalse();
+
+        view.Entries.ShouldHaveSingleItem().Snapshot.State.ShouldBe("Healthy");
+    }
+
+    [Fact]
+    public void Staleness_is_per_node()
+    {
+        var view = new DriverStatusByNode("drv");
+        view.Upsert(Snap("drv", "Healthy", T0, "central-1"), receivedUtc: T0);
+        view.Upsert(Snap("drv", "Healthy", T0, "central-2"), receivedUtc: T0.AddSeconds(40));
+
+        var now = T0.AddSeconds(45);
+        var stale = TimeSpan.FromSeconds(30);
+        var entries = view.Entries;
+        entries.Single(e => e.NodeId == "central-1").IsStale(now, stale).ShouldBeTrue();
+        entries.Single(e => e.NodeId == "central-2").IsStale(now, stale).ShouldBeFalse();
+    }
+
+    [Fact]
+    public void Priming_from_the_store_picks_up_every_node_of_the_driver()
+    {
+        // The store keys per (driver, node) since #847; the panel primes from GetAll(), not TryGet() (which
+        // answers only the most recent across nodes).
+        var store = new InMemoryDriverStatusSnapshotStore();
+        store.Upsert(Snap("drv", "Faulted", T0, "central-1", "boom"));
+        store.Upsert(Snap("drv", "Healthy", T0.AddSeconds(1), "central-2"));
+        store.Upsert(Snap("other", "Healthy", T0, "central-1"));
+
+        var view = new DriverStatusByNode("drv");
+        foreach (var s in store.GetAll()) view.Upsert(s, T0);
+
+        view.Entries.Select(e => $"{e.NodeId}:{e.Snapshot.State}").ShouldBe(["central-1:Faulted", "central-2:Healthy"]);
+    }
+
+    [Fact]
+    public void The_status_panel_renders_per_node_entries_and_primes_from_GetAll()
+    {
+        var src = AdminUiSource.Read("Components", "Shared", "Drivers", "DriverStatusPanel.razor");
+
+        src.ShouldContain("new DriverStatusByNode(DriverInstanceId)");
+        src.ShouldContain("StatusStore.GetAll()");
+        src.ShouldNotContain("StatusStore.TryGet(");
+        src.ShouldContain("@foreach (var n in nodes)");
+        src.ShouldContain("@n.Label");
+        src.ShouldContain("@key=\"@($\"node-{n.Key}\")\"");
+        // The old single-snapshot field must be gone, or a half-migrated panel could still render one node.
+        src.ShouldNotContain("_snapshot");
+    }
+}