feat(telemetry): wire TelemetryDial:CallTimeoutSeconds as the dial-establishment deadline (A5)

CallTimeoutSeconds was bound, defaulted and documented but consumed
nowhere. TelemetryDialSupervisor now arms a per-generation deadline on
every dial: a dial that has delivered no event by the deadline is
cancelled and treated as a transport failure (reconnect with a failure
streak). A live node replays its snapshots on subscribe, so this bites
only a dial that never connects (a black-holed host, where the socket
connect has no timeout). An established stream is deliberately not
bounded — the wire has no heartbeat, keepalive pings own its liveness.

Review ref: 02/U2.

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:38:32 -04:00
parent 79d6b0a4b2
commit c6d4b14c93
3 changed files with 200 additions and 1 deletions
@@ -129,6 +129,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
Receive<TelemetryReceived>(HandleTelemetryReceived);
Receive<StreamStopped>(HandleStreamStopped);
Receive<Reconnect>(HandleReconnect);
Receive<DialDeadline>(HandleDialDeadline);
// A faulted node-source task pipes here. Log and wait for the next periodic refresh — a stale
// node set is far better than crashing the supervisor and dropping every live dialer.
@@ -294,6 +295,34 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
}
}
/// <summary>
/// <c>TelemetryDial:CallTimeoutSeconds</c>: a dial that has delivered no event by the deadline is cut and
/// treated as a transport failure (reconnect with a failure streak). <c>Connected</c> flips on the first
/// routed event, on this thread, so a dial that connected in time is left alone — the deadline bounds
/// ESTABLISHMENT only. A live node answers a fresh subscribe at once (the node-local hub replays its
/// health/resilience snapshots), so what this catches is a dial that never connects: a black-holed host,
/// where the socket connect itself has no timeout and the stream would otherwise hang forever without a
/// reconnect. An established stream is never bounded here — the wire has no heartbeat, keepalive pings own
/// its liveness, and an idle cut would churn every quiet node.
/// </summary>
private void HandleDialDeadline(DialDeadline msg)
{
if (!_dialers.TryGetValue(msg.NodeId, out var dialer) || msg.Generation != dialer.Generation || dialer.Connected)
{
return;
}
_log.Warning(
"Telemetry dial to node {NodeId} ({Endpoint}) delivered nothing within {Deadline}s; cancelling it and re-dialing",
msg.NodeId, dialer.Target.Endpoint, DialDeadlineFor().TotalSeconds);
dialer.Cancel();
HandleStreamStopped(new StreamStopped(
msg.NodeId, msg.Generation,
new TimeoutException($"no telemetry event within {DialDeadlineFor().TotalSeconds:F0}s of dialing")));
}
private TimeSpan DialDeadlineFor() => TimeSpan.FromSeconds(Math.Max(1, _options.CallTimeoutSeconds));
private void StartDialer(string nodeId)
{
var dialer = _dialers[nodeId];
@@ -306,6 +335,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
var cts = new CancellationTokenSource();
dialer.Cts = cts;
dialer.Connected = false;
// One deadline per generation; a superseded tick is dropped by the generation guard in the handler.
Timers.StartSingleTimer(DeadlineTimerKey(nodeId), new DialDeadline(nodeId, generation), DialDeadlineFor());
var self = Self;
var target = dialer.Target;
@@ -357,6 +388,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
{
dialer.Cancel();
Timers.Cancel(ReconnectTimerKey(nodeId));
Timers.Cancel(DeadlineTimerKey(nodeId));
RecomputePill();
}
}
@@ -433,6 +465,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
private static string ReconnectTimerKey(string nodeId) => $"telemetry-reconnect-{nodeId}";
private static string DeadlineTimerKey(string nodeId) => $"telemetry-dial-deadline-{nodeId}";
/// <summary>Per-node dialer bookkeeping. Mutated only on the actor thread.</summary>
private sealed class NodeDialer(TelemetryDialTarget target)
{
@@ -496,4 +530,10 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
/// <summary>Backoff timer tick: start a fresh dialer for the node.</summary>
/// <param name="NodeId">The node id to reconnect.</param>
public sealed record Reconnect(string NodeId);
/// <summary>Dial-establishment deadline tick (<c>TelemetryDial:CallTimeoutSeconds</c>); see
/// <see cref="HandleDialDeadline"/>.</summary>
/// <param name="NodeId">The node whose dial is being checked.</param>
/// <param name="Generation">The dialer generation the deadline was armed for.</param>
public sealed record DialDeadline(string NodeId, int Generation);
}