feat(telemetry): wire TelemetryDial:CallTimeoutSeconds as the dial-establishment deadline (A5)

CallTimeoutSeconds was bound, defaulted and documented but consumed
nowhere. TelemetryDialSupervisor now arms a per-generation deadline on
every dial: a dial that has delivered no event by the deadline is
cancelled and treated as a transport failure (reconnect with a failure
streak). A live node replays its snapshots on subscribe, so this bites
only a dial that never connects (a black-holed host, where the socket
connect has no timeout). An established stream is deliberately not
bounded — the wire has no heartbeat, keepalive pings own its liveness.

Review ref: 02/U2.

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Core/ZB.MOM.WW.OtOpcUa.Cluster/TelemetryOptions.cs

@@ -72,7 +72,13 @@ public sealed class TelemetryDialOptions
     /// <summary>How often central refreshes its set of dialable node contacts, in seconds.</summary>
     public int ContactRefreshSeconds { get; set; } = 60;
 
-    /// <summary>Per-call deadline for a gRPC telemetry dial, in seconds.</summary>
+    /// <summary>
+    ///     Deadline, in seconds, for a gRPC telemetry dial to deliver its first event. A live node answers a
+    ///     fresh subscribe at once (it replays its health/resilience snapshots), so a dial still silent at the
+    ///     deadline is one that never connected — it is cancelled and re-dialled. Bounds establishment only;
+    ///     an established stream is kept alive by HTTP/2 keepalive pings, not by this value. Values below 1
+    ///     are treated as 1.
+    /// </summary>
     public int CallTimeoutSeconds { get; set; } = 30;
 }
 

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Telemetry/TelemetryDialSupervisor.cs

@@ -129,6 +129,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         Receive<TelemetryReceived>(HandleTelemetryReceived);
         Receive<StreamStopped>(HandleStreamStopped);
         Receive<Reconnect>(HandleReconnect);
+        Receive<DialDeadline>(HandleDialDeadline);
 
         // A faulted node-source task pipes here. Log and wait for the next periodic refresh — a stale
         // node set is far better than crashing the supervisor and dropping every live dialer.
@@ -294,6 +295,34 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    ///     <c>TelemetryDial:CallTimeoutSeconds</c>: a dial that has delivered no event by the deadline is cut and
+    ///     treated as a transport failure (reconnect with a failure streak). <c>Connected</c> flips on the first
+    ///     routed event, on this thread, so a dial that connected in time is left alone — the deadline bounds
+    ///     ESTABLISHMENT only. A live node answers a fresh subscribe at once (the node-local hub replays its
+    ///     health/resilience snapshots), so what this catches is a dial that never connects: a black-holed host,
+    ///     where the socket connect itself has no timeout and the stream would otherwise hang forever without a
+    ///     reconnect. An established stream is never bounded here — the wire has no heartbeat, keepalive pings own
+    ///     its liveness, and an idle cut would churn every quiet node.
+    /// </summary>
+    private void HandleDialDeadline(DialDeadline msg)
+    {
+        if (!_dialers.TryGetValue(msg.NodeId, out var dialer) || msg.Generation != dialer.Generation || dialer.Connected)
+        {
+            return;
+        }
+
+        _log.Warning(
+            "Telemetry dial to node {NodeId} ({Endpoint}) delivered nothing within {Deadline}s; cancelling it and re-dialing",
+            msg.NodeId, dialer.Target.Endpoint, DialDeadlineFor().TotalSeconds);
+        dialer.Cancel();
+        HandleStreamStopped(new StreamStopped(
+            msg.NodeId, msg.Generation,
+            new TimeoutException($"no telemetry event within {DialDeadlineFor().TotalSeconds:F0}s of dialing")));
+    }
+
+    private TimeSpan DialDeadlineFor() => TimeSpan.FromSeconds(Math.Max(1, _options.CallTimeoutSeconds));
+
     private void StartDialer(string nodeId)
     {
         var dialer = _dialers[nodeId];
@@ -306,6 +335,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         var cts = new CancellationTokenSource();
         dialer.Cts = cts;
         dialer.Connected = false;
+        // One deadline per generation; a superseded tick is dropped by the generation guard in the handler.
+        Timers.StartSingleTimer(DeadlineTimerKey(nodeId), new DialDeadline(nodeId, generation), DialDeadlineFor());
 
         var self = Self;
         var target = dialer.Target;
@@ -357,6 +388,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
         {
             dialer.Cancel();
             Timers.Cancel(ReconnectTimerKey(nodeId));
+            Timers.Cancel(DeadlineTimerKey(nodeId));
             RecomputePill();
         }
     }
@@ -433,6 +465,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
 
     private static string ReconnectTimerKey(string nodeId) => $"telemetry-reconnect-{nodeId}";
 
+    private static string DeadlineTimerKey(string nodeId) => $"telemetry-dial-deadline-{nodeId}";
+
     /// <summary>Per-node dialer bookkeeping. Mutated only on the actor thread.</summary>
     private sealed class NodeDialer(TelemetryDialTarget target)
     {
@@ -496,4 +530,10 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
     /// <summary>Backoff timer tick: start a fresh dialer for the node.</summary>
     /// <param name="NodeId">The node id to reconnect.</param>
     public sealed record Reconnect(string NodeId);
+
+    /// <summary>Dial-establishment deadline tick (<c>TelemetryDial:CallTimeoutSeconds</c>); see
+    /// <see cref="HandleDialDeadline"/>.</summary>
+    /// <param name="NodeId">The node whose dial is being checked.</param>
+    /// <param name="Generation">The dialer generation the deadline was armed for.</param>
+    public sealed record DialDeadline(string NodeId, int Generation);
 }