Merge master into remed/lane-F

Refs #735

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Clients/FleetDiagnosticsClient.cs

@@ -1,4 +1,6 @@
 using Akka.Actor;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
@@ -13,7 +15,9 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
 /// Sends <see cref="GetDiagnostics"/>; expects a <see cref="NodeDiagnosticsSnapshot"/> reply.
 ///
 /// On timeout or any other failure (peer down, GetDiagnostics handler missing) returns an
-/// empty snapshot so the UI degrades to "no data" instead of throwing.
+/// empty snapshot so the UI degrades to "no data" instead of throwing. The failure is logged at
+/// Warning, since the snapshot shape cannot carry it: an empty snapshot reads the same as a node with
+/// no drivers. Consumed by the <c>/fleet/diagnostics</c> page.
 /// </summary>
 public sealed class FleetDiagnosticsClient : IFleetDiagnosticsClient
 {
@@ -21,29 +25,38 @@ public sealed class FleetDiagnosticsClient : IFleetDiagnosticsClient
 
     private readonly ActorSystem _system;
     private readonly string _systemName;
+    private readonly ILogger<FleetDiagnosticsClient> _log;
 
     /// <summary>Initializes a new FleetDiagnosticsClient with the given actor system and cluster options.</summary>
     /// <param name="system">The Akka actor system.</param>
     /// <param name="options">Cluster configuration options.</param>
-    public FleetDiagnosticsClient(ActorSystem system, IOptions<AkkaClusterOptions> options)
+    /// <param name="log">Logger for the degraded-to-empty path; defaults to a no-op logger.</param>
+    public FleetDiagnosticsClient(
+        ActorSystem system, IOptions<AkkaClusterOptions> options, ILogger<FleetDiagnosticsClient>? log = null)
     {
         _system = system;
         _systemName = options.Value.SystemName;
+        _log = log ?? NullLogger<FleetDiagnosticsClient>.Instance;
     }
 
     /// <inheritdoc />
     public async Task<NodeDiagnosticsSnapshot> GetDiagnosticsAsync(NodeId nodeId, CancellationToken ct)
     {
-        var selection = _system.ActorSelection($"akka.tcp://{_systemName}@{nodeId.Value}/user/driver-host");
         try
         {
+            // Inside the try: a malformed node value makes ActorSelection throw, and that must degrade to
+            // "unavailable" like a dead node does, not surface through the page's error boundary.
+            var selection = _system.ActorSelection($"akka.tcp://{_systemName}@{nodeId.Value}/user/driver-host");
             using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
             linked.CancelAfter(AskTimeout);
             return await selection.Ask<NodeDiagnosticsSnapshot>(
                 new GetDiagnostics(CorrelationId.NewId()), AskTimeout, linked.Token);
         }
-        catch (Exception)
+        catch (Exception ex) when (!ct.IsCancellationRequested)
         {
+            _log.LogWarning(ex,
+                "Fleet diagnostics for node {NodeId} unavailable (no answer within {Timeout}s); returning an empty snapshot",
+                nodeId.Value, AskTimeout.TotalSeconds);
             return new NodeDiagnosticsSnapshot(
                 nodeId,
                 CurrentRevision: null,

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/FleetDiagnostics.razor

@@ -0,0 +1,238 @@
+@page "/fleet/diagnostics"
+@* Per-node driver diagnostics, fetched on demand from the selected driver node's DriverHostActor over
+   Akka (IFleetDiagnosticsClient → GetDiagnostics → NodeDiagnosticsSnapshot). The node answers from
+   its live child set + each child's last-published health; the "running from cache" flag is the one
+   fact /hosts cannot show (a node booted from its artifact cache looks healthy but is frozen).
+   Reach it as /fleet/diagnostics?node=<host:port> (the NodeId shown on /fleet and /hosts), or pick
+   a driver-role member from the selector. *@
+@attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
+@rendermode RenderMode.InteractiveServer
+@using ZB.MOM.WW.OtOpcUa.Cluster
+@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
+@using ZB.MOM.WW.OtOpcUa.Commons.Types
+@inject IClusterRoleInfo Cluster
+@inject IFleetDiagnosticsClient Diagnostics
+@inject NavigationManager Nav
+@implements IDisposable
+
+<PageErrorBoundary>
+<div class="d-flex justify-content-between align-items-center mb-3">
+    <h4 class="mb-0">Node diagnostics</h4>
+    <a class="btn btn-sm btn-outline-secondary" href="/fleet">Back to fleet status</a>
+</div>
+
+<div class="d-flex align-items-center mb-3 gap-2 flex-wrap">
+    <label class="small text-muted mb-0" for="diag-node">Node</label>
+    <select id="diag-node" class="form-select form-select-sm" style="width:auto"
+            value="@_selectedNode" @onchange="OnNodeSelected">
+        <option value="">— select a driver node —</option>
+        @foreach (var member in _members)
+        {
+            <option value="@member">@member</option>
+        }
+        @if (!string.IsNullOrEmpty(_selectedNode) && !_members.Contains(_selectedNode))
+        {
+            <option value="@_selectedNode">@_selectedNode (not an Up member)</option>
+        }
+    </select>
+    <input type="text" class="form-control form-control-sm" style="width:220px"
+           placeholder="or type host:port…" @bind="_typedNode" @bind:event="oninput"
+           @onkeydown="OnTypedKeyDown" />
+    <button class="btn btn-sm btn-outline-primary" @onclick="UseTypedNodeAsync"
+            disabled="@(string.IsNullOrWhiteSpace(_typedNode))">Go</button>
+    <button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync"
+            disabled="@(_loading || string.IsNullOrEmpty(_selectedNode))">
+        @if (_loading) { <span class="spinner-border spinner-border-sm me-1" /> }
+        Refresh
+    </button>
+    <span class="text-muted small">
+        Last fetched: @(_fetchedUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
+    </span>
+</div>
+
+@if (string.IsNullOrEmpty(_selectedNode))
+{
+    <section class="panel notice rise" style="animation-delay:.02s">
+        Pick a driver-role node to fetch its driver diagnostics. The node answers from its own
+        DriverHostActor, so this works for any node reachable over Akka remoting from this admin node.
+    </section>
+}
+else if (_snapshot is null)
+{
+    <p>@(_loading ? "Fetching…" : "No snapshot yet.")</p>
+}
+else
+{
+    <section class="agg-grid rise" style="animation-delay:.02s">
+        <div class="agg-card">
+            <div class="agg-label">Node</div>
+            <div class="agg-value mono">@_snapshot.NodeId.Value</div>
+        </div>
+        <div class="agg-card">
+            <div class="agg-label">Revision</div>
+            <div class="agg-value mono" title="@(_snapshot.CurrentRevision?.Value ?? "")">
+                @(_snapshot.CurrentRevision is { } rev ? DisplayText.Abbreviate(rev.Value, 12) : "—")
+            </div>
+        </div>
+        <div class="agg-card @(_snapshot.RunningFromCache ? "caution" : "")">
+            <div class="agg-label">Config source</div>
+            <div class="agg-value">@(_snapshot.RunningFromCache ? "cache (frozen)" : "live")</div>
+        </div>
+        <div class="agg-card">
+            <div class="agg-label">Drivers</div>
+            <div class="agg-value numeric">@_snapshot.Drivers.Count</div>
+        </div>
+        <div class="agg-card alert">
+            <div class="agg-label">Faulted devices</div>
+            <div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
+        </div>
+    </section>
+
+    @if (_snapshot.RunningFromCache)
+    {
+        <section class="panel notice rise" style="animation-delay:.05s">
+            This node booted its configuration from its local artifact cache because central was unreachable.
+            It serves its last-known-good address space, but no deployment can reach it until central is back.
+        </section>
+    }
+
+    <section class="panel rise" style="animation-delay:.08s">
+        <div class="panel-head">
+            Drivers
+            <span class="text-muted small ms-2">as of @_snapshot.AsOfUtc.ToString("u")</span>
+        </div>
+        @if (_snapshot.Drivers.Count == 0)
+        {
+            <div class="p-3 text-muted">
+                Empty snapshot: the node has no driver children — or it did not answer within the client's
+                3 s budget (an unreachable node degrades to an empty snapshot; see the AdminUI log for the
+                FleetDiagnosticsClient warning).
+            </div>
+        }
+        else
+        {
+            <div class="table-wrap">
+                <table class="data-table">
+                    <thead>
+                        <tr>
+                            <th>Driver instance</th>
+                            <th>Row id</th>
+                            <th>State</th>
+                            <th class="numeric">Connected devices</th>
+                            <th class="numeric">Faulted devices</th>
+                            <th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        @foreach (var d in _snapshot.Drivers.OrderBy(d => d.Name, StringComparer.OrdinalIgnoreCase))
+                        {
+                            <tr>
+                                <td>@d.Name</td>
+                                <td><span class="mono" title="@d.DriverInstanceId">@(d.DriverInstanceId == Guid.Empty ? "—" : DisplayText.Abbreviate(d.DriverInstanceId.ToString(), 13))</span></td>
+                                <td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
+                                <td class="numeric">@d.ConnectedDevices</td>
+                                <td class="numeric">@d.FaultedDevices</td>
+                                <td>@d.LastChangeUtc.ToString("u")</td>
+                            </tr>
+                        }
+                    </tbody>
+                </table>
+            </div>
+        }
+    </section>
+}
+</PageErrorBoundary>
+
+@code {
+    [SupplyParameterFromQuery(Name = "node")]
+    public string? Node { get; set; }
+
+    private IReadOnlyList<string> _members = Array.Empty<string>();
+    private string _selectedNode = "";
+    private string _typedNode = "";
+    private NodeDiagnosticsSnapshot? _snapshot;
+    private bool _loading;
+    private DateTime? _fetchedUtc;
+
+    // Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
+    private readonly CancellationTokenSource _cts = new();
+    private volatile bool _disposed;
+
+    protected override async Task OnParametersSetAsync()
+    {
+        _members = Cluster.MembersWithRole(RoleParser.Driver)
+            .Select(n => n.Value)
+            .OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+        var requested = Node?.Trim() ?? "";
+        if (!string.Equals(requested, _selectedNode, StringComparison.Ordinal))
+        {
+            _selectedNode = requested;
+            _snapshot = null;
+            if (!string.IsNullOrEmpty(_selectedNode)) await LoadAsync();
+        }
+    }
+
+    private Task OnNodeSelected(ChangeEventArgs e)
+    {
+        var value = e.Value?.ToString() ?? "";
+        Navigate(value);
+        return Task.CompletedTask;
+    }
+
+    private Task UseTypedNodeAsync()
+    {
+        Navigate(_typedNode.Trim());
+        return Task.CompletedTask;
+    }
+
+    private Task OnTypedKeyDown(KeyboardEventArgs e)
+    {
+        if (e.Key == "Enter") Navigate(_typedNode.Trim());
+        return Task.CompletedTask;
+    }
+
+    // The node id lives in the URL so a diagnostics view is linkable from /fleet and survives a reload.
+    private void Navigate(string node) =>
+        Nav.NavigateTo(string.IsNullOrEmpty(node)
+            ? "/fleet/diagnostics"
+            : "/fleet/diagnostics?node=" + Uri.EscapeDataString(node));
+
+    private async Task RefreshAsync() => await LoadAsync();
+
+    private async Task LoadAsync()
+    {
+        if (_loading || string.IsNullOrEmpty(_selectedNode)) return;
+        _loading = true;
+        try
+        {
+            // The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
+            _snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
+            _fetchedUtc = DateTime.UtcNow;
+        }
+        catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
+        {
+            // Circuit gone mid-fetch — a clean exit, not a fault.
+        }
+        finally
+        {
+            _loading = false;
+            if (!_disposed) StateHasChanged();
+        }
+    }
+
+    private static string StateChipClass(string state) => state switch
+    {
+        "Healthy" => "chip-ok",
+        "Degraded" or "Connecting" or "Reconnecting" => "chip-caution",
+        "Faulted" => "chip-alert",
+        _ => "chip-idle",
+    };
+
+    public void Dispose()
+    {
+        _disposed = true;
+        _cts.Cancel();
+        _cts.Dispose();
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Telemetry/TelemetryDialSupervisor.cs

@@ -129,6 +129,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         Receive<TelemetryReceived>(HandleTelemetryReceived);
         Receive<StreamStopped>(HandleStreamStopped);
         Receive<Reconnect>(HandleReconnect);
+        Receive<DialDeadline>(HandleDialDeadline);
 
         // A faulted node-source task pipes here. Log and wait for the next periodic refresh — a stale
         // node set is far better than crashing the supervisor and dropping every live dialer.
@@ -251,6 +252,9 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
             return;
         }
 
+        // The stream is over; a still-armed establishment deadline for it must not fire into the reconnect.
+        Timers.Cancel(DeadlineTimerKey(msg.NodeId));
+
         if (dialer.Connected)
         {
             dialer.Connected = false;
@@ -294,6 +298,34 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    ///     <c>TelemetryDial:CallTimeoutSeconds</c>: a dial that has delivered no event by the deadline is cut and
+    ///     treated as a transport failure (reconnect with a failure streak). <c>Connected</c> flips on the first
+    ///     routed event, on this thread, so a dial that connected in time is left alone — the deadline bounds
+    ///     ESTABLISHMENT only. A live node answers a fresh subscribe at once (the node-local hub replays its
+    ///     health/resilience snapshots), so what this catches is a dial that never connects: a black-holed host,
+    ///     where the socket connect itself has no timeout and the stream would otherwise hang forever without a
+    ///     reconnect. An established stream is never bounded here — the wire has no heartbeat, keepalive pings own
+    ///     its liveness, and an idle cut would churn every quiet node.
+    /// </summary>
+    private void HandleDialDeadline(DialDeadline msg)
+    {
+        if (!_dialers.TryGetValue(msg.NodeId, out var dialer) || msg.Generation != dialer.Generation || dialer.Connected)
+        {
+            return;
+        }
+
+        _log.Warning(
+            "Telemetry dial to node {NodeId} ({Endpoint}) delivered nothing within {Deadline}s; cancelling it and re-dialing",
+            msg.NodeId, dialer.Target.Endpoint, DialDeadlineFor().TotalSeconds);
+        dialer.Cancel();
+        HandleStreamStopped(new StreamStopped(
+            msg.NodeId, msg.Generation,
+            new TimeoutException($"no telemetry event within {DialDeadlineFor().TotalSeconds:F0}s of dialing")));
+    }
+
+    private TimeSpan DialDeadlineFor() => TimeSpan.FromSeconds(Math.Max(1, _options.CallTimeoutSeconds));
+
     private void StartDialer(string nodeId)
     {
         var dialer = _dialers[nodeId];
@@ -306,6 +338,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         var cts = new CancellationTokenSource();
         dialer.Cts = cts;
         dialer.Connected = false;
+        // One deadline per generation; a superseded tick is dropped by the generation guard in the handler.
+        Timers.StartSingleTimer(DeadlineTimerKey(nodeId), new DialDeadline(nodeId, generation), DialDeadlineFor());
 
         var self = Self;
         var target = dialer.Target;
@@ -357,6 +391,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         {
             dialer.Cancel();
             Timers.Cancel(ReconnectTimerKey(nodeId));
+            Timers.Cancel(DeadlineTimerKey(nodeId));
             RecomputePill();
         }
     }
@@ -433,6 +468,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
 
     private static string ReconnectTimerKey(string nodeId) => $"telemetry-reconnect-{nodeId}";
 
+    private static string DeadlineTimerKey(string nodeId) => $"telemetry-dial-deadline-{nodeId}";
+
     /// <summary>Per-node dialer bookkeeping. Mutated only on the actor thread.</summary>
     private sealed class NodeDialer(TelemetryDialTarget target)
     {
@@ -496,4 +533,10 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
     /// <summary>Backoff timer tick: start a fresh dialer for the node.</summary>
     /// <param name="NodeId">The node id to reconnect.</param>
     public sealed record Reconnect(string NodeId);
+
+    /// <summary>Dial-establishment deadline tick (<c>TelemetryDial:CallTimeoutSeconds</c>); see
+    /// <see cref="HandleDialDeadline"/>.</summary>
+    /// <param name="NodeId">The node whose dial is being checked.</param>
+    /// <param name="Generation">The dialer generation the deadline was armed for.</param>
+    public sealed record DialDeadline(string NodeId, int Generation);
 }

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/ZB.MOM.WW.OtOpcUa.AdminUI.csproj

@@ -9,6 +9,12 @@
     <ItemGroup>
         <FrameworkReference Include="Microsoft.AspNetCore.App"/>
         <PackageReference Include="Microsoft.AspNetCore.SignalR.Client"/>
+        <!-- Surgical transitive pin — see the Microsoft.AspNetCore.SignalR.Common block in
+             Directory.Packages.props (GHSA-334f-vvp4-7g6p on SignalR.Common 10.0.0). Inside AdminUI
+             the shared framework already supplies it (hence NU1510), but the PackageReference flows
+             to the non-Web consumers (Host.Tests, AdminUI.Tests, OpcUaServer.IntegrationTests) where
+             SignalR.Client 10.0.0 would otherwise resolve the vulnerable 10.0.0. -->
+        <PackageReference Include="Microsoft.AspNetCore.SignalR.Common" NoWarn="NU1510"/>
         <PackageReference Include="ZB.MOM.WW.Theme"/>
         <PackageReference Include="ZB.MOM.WW.Secrets.Ui"/>
     </ItemGroup>