feat(site-runtime): tag-subscribe failure reply frees the in-flight slot, backs off (×2 ±20%, capped) and Success resets to floor (Lever 1)

Also from the Task 3 review: FailedSubscribeResponse_IsRetried guards the failure path again (large ceiling); HandleSubscribeTagsResponse reads state instead of get-or-create; SubscribeToDcl prunes stale retry state on re-entry; doc comments refreshed.

Claude-Session: https://claude.ai/code/session_019ZxXT5sxZs5XrtdwYzJDPV
This commit is contained in:
Joseph Doherty
2026-08-28 14:36:22 -04:00
parent 029d39fca2
commit b6b401c879
2 changed files with 95 additions and 10 deletions
@@ -186,6 +186,8 @@ public class InstanceActor : ReceiveActor
// (a tick while a young request is outstanding re-arms instead of re-sending) and
// exponential backoff (see NextTagSubscribeInterval) so a slow gateway is never
// handed a fresh N-tag subscribe on top of the one it is still processing.
// A re-entry into SubscribeToDcl rebuilds all three together: the retry state is
// pruned (timers cancelled, pending rows dropped) before the new grouping is built.
private readonly Dictionary<string, List<string>> _tagsByConnection = new();
private readonly Dictionary<string, string> _pendingTagSubscribes = new();
private readonly Dictionary<string, TagSubscribeState> _tagSubscribeStates = new();
@@ -1206,6 +1208,16 @@ public class InstanceActor : ReceiveActor
{
if (_dclManager == null || _configuration == null) return;
// Lever 1: prune retry state from a previous build — cancels any armed watchdog and
// drops its pending row so a re-entry can never strand an in-flight slot.
foreach (var state in _tagSubscribeStates.Values)
{
state.Timer?.Cancel();
if (state.InFlightCorrelationId != null)
_pendingTagSubscribes.Remove(state.InFlightCorrelationId);
}
_tagSubscribeStates.Clear();
// Group attributes by their bound connection name into the persistent
// _tagsByConnection field so a retry can re-send the exact same request.
_tagsByConnection.Clear();
@@ -1348,18 +1360,27 @@ public class InstanceActor : ReceiveActor
state.Interval = TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs);
/// <summary>
/// Handles the DCL's tag-subscribe response: on Success cancels the connection's
/// retry timer; on failure logs + emits a site event and leaves the armed timer
/// to re-send. A reply for an unknown/superseded correlation id is ignored.
/// Handles the DCL's tag-subscribe response. A reply whose correlation id is not the
/// pending one (superseded at the in-flight ceiling, or a duplicate) is ignored. On
/// Success the connection's watchdog is cancelled, the in-flight slot cleared and the
/// backoff reset to the floor. On failure (Lever 1, 2026-08-28) the slot is freed, the
/// backoff advanced (×2 ±20 % jitter, capped) and the watchdog re-armed at the new
/// interval, so the next attempt waits the backed-off interval — not the in-flight
/// ceiling — and duplicates are still impossible.
/// </summary>
private void HandleSubscribeTagsResponse(SubscribeTagsResponse response)
{
if (!_pendingTagSubscribes.Remove(response.CorrelationId, out var connectionName))
return; // stale/superseded reply — ignore
// Read, never create: the id was minted by SendTagSubscribe, which creates the
// state, so a missing entry means the retry state was pruned (SubscribeToDcl
// re-entry) and this reply no longer governs anything.
if (!_tagSubscribeStates.TryGetValue(connectionName, out var state))
return;
if (response.Success)
{
var state = GetTagSubscribeState(connectionName);
if (state.InFlightCorrelationId == response.CorrelationId)
{
state.Timer?.Cancel();
@@ -1373,14 +1394,21 @@ public class InstanceActor : ReceiveActor
}
else
{
if (state.InFlightCorrelationId == response.CorrelationId)
{
// A definite failure: the slot is free, back off, and re-arm at the NEW
// interval so the backoff is honoured on this very retry, not the one after.
state.InFlightCorrelationId = null;
BackOffTagSubscribe(state);
ArmTagSubscribeWatchdog(connectionName, state);
}
_logger.LogWarning(
"Instance {Instance} tag subscribe FAILED on connection {Connection}: {Error} — will retry",
_instanceUniqueName, connectionName, response.ErrorMessage);
"Instance {Instance} tag subscribe FAILED on connection {Connection}: {Error} — will retry in {Interval}",
_instanceUniqueName, connectionName, response.ErrorMessage, state.Interval);
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"instance_lifecycle", "Warning", _instanceUniqueName,
$"InstanceActor:{_instanceUniqueName}",
$"Tag subscribe failed on connection '{connectionName}': {response.ErrorMessage} — will retry");
// Leave the armed retry timer to re-send.
$"Tag subscribe failed on connection '{connectionName}': {response.ErrorMessage} — will retry in {state.Interval}");
}
}