feat(site-runtime): tag-subscribe failure reply frees the in-flight slot, backs off (×2 ±20%, capped) and Success resets to floor (Lever 1)
Also from the Task 3 review: FailedSubscribeResponse_IsRetried guards the failure path again (large ceiling); HandleSubscribeTagsResponse reads state instead of get-or-create; SubscribeToDcl prunes stale retry state on re-entry; doc comments refreshed. Claude-Session: https://claude.ai/code/session_019ZxXT5sxZs5XrtdwYzJDPV
This commit is contained in:
@@ -186,6 +186,8 @@ public class InstanceActor : ReceiveActor
|
||||
// (a tick while a young request is outstanding re-arms instead of re-sending) and
|
||||
// exponential backoff (see NextTagSubscribeInterval) so a slow gateway is never
|
||||
// handed a fresh N-tag subscribe on top of the one it is still processing.
|
||||
// A re-entry into SubscribeToDcl rebuilds all three together: the retry state is
|
||||
// pruned (timers cancelled, pending rows dropped) before the new grouping is built.
|
||||
private readonly Dictionary<string, List<string>> _tagsByConnection = new();
|
||||
private readonly Dictionary<string, string> _pendingTagSubscribes = new();
|
||||
private readonly Dictionary<string, TagSubscribeState> _tagSubscribeStates = new();
|
||||
@@ -1206,6 +1208,16 @@ public class InstanceActor : ReceiveActor
|
||||
{
|
||||
if (_dclManager == null || _configuration == null) return;
|
||||
|
||||
// Lever 1: prune retry state from a previous build — cancels any armed watchdog and
|
||||
// drops its pending row so a re-entry can never strand an in-flight slot.
|
||||
foreach (var state in _tagSubscribeStates.Values)
|
||||
{
|
||||
state.Timer?.Cancel();
|
||||
if (state.InFlightCorrelationId != null)
|
||||
_pendingTagSubscribes.Remove(state.InFlightCorrelationId);
|
||||
}
|
||||
_tagSubscribeStates.Clear();
|
||||
|
||||
// Group attributes by their bound connection name into the persistent
|
||||
// _tagsByConnection field so a retry can re-send the exact same request.
|
||||
_tagsByConnection.Clear();
|
||||
@@ -1348,18 +1360,27 @@ public class InstanceActor : ReceiveActor
|
||||
state.Interval = TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs);
|
||||
|
||||
/// <summary>
|
||||
/// Handles the DCL's tag-subscribe response: on Success cancels the connection's
|
||||
/// retry timer; on failure logs + emits a site event and leaves the armed timer
|
||||
/// to re-send. A reply for an unknown/superseded correlation id is ignored.
|
||||
/// Handles the DCL's tag-subscribe response. A reply whose correlation id is not the
|
||||
/// pending one (superseded at the in-flight ceiling, or a duplicate) is ignored. On
|
||||
/// Success the connection's watchdog is cancelled, the in-flight slot cleared and the
|
||||
/// backoff reset to the floor. On failure (Lever 1, 2026-08-28) the slot is freed, the
|
||||
/// backoff advanced (×2 ±20 % jitter, capped) and the watchdog re-armed at the new
|
||||
/// interval, so the next attempt waits the backed-off interval — not the in-flight
|
||||
/// ceiling — and duplicates are still impossible.
|
||||
/// </summary>
|
||||
private void HandleSubscribeTagsResponse(SubscribeTagsResponse response)
|
||||
{
|
||||
if (!_pendingTagSubscribes.Remove(response.CorrelationId, out var connectionName))
|
||||
return; // stale/superseded reply — ignore
|
||||
|
||||
// Read, never create: the id was minted by SendTagSubscribe, which creates the
|
||||
// state, so a missing entry means the retry state was pruned (SubscribeToDcl
|
||||
// re-entry) and this reply no longer governs anything.
|
||||
if (!_tagSubscribeStates.TryGetValue(connectionName, out var state))
|
||||
return;
|
||||
|
||||
if (response.Success)
|
||||
{
|
||||
var state = GetTagSubscribeState(connectionName);
|
||||
if (state.InFlightCorrelationId == response.CorrelationId)
|
||||
{
|
||||
state.Timer?.Cancel();
|
||||
@@ -1373,14 +1394,21 @@ public class InstanceActor : ReceiveActor
|
||||
}
|
||||
else
|
||||
{
|
||||
if (state.InFlightCorrelationId == response.CorrelationId)
|
||||
{
|
||||
// A definite failure: the slot is free, back off, and re-arm at the NEW
|
||||
// interval so the backoff is honoured on this very retry, not the one after.
|
||||
state.InFlightCorrelationId = null;
|
||||
BackOffTagSubscribe(state);
|
||||
ArmTagSubscribeWatchdog(connectionName, state);
|
||||
}
|
||||
_logger.LogWarning(
|
||||
"Instance {Instance} tag subscribe FAILED on connection {Connection}: {Error} — will retry",
|
||||
_instanceUniqueName, connectionName, response.ErrorMessage);
|
||||
"Instance {Instance} tag subscribe FAILED on connection {Connection}: {Error} — will retry in {Interval}",
|
||||
_instanceUniqueName, connectionName, response.ErrorMessage, state.Interval);
|
||||
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
|
||||
"instance_lifecycle", "Warning", _instanceUniqueName,
|
||||
$"InstanceActor:{_instanceUniqueName}",
|
||||
$"Tag subscribe failed on connection '{connectionName}': {response.ErrorMessage} — will retry");
|
||||
// Leave the armed retry timer to re-send.
|
||||
$"Tag subscribe failed on connection '{connectionName}': {response.ErrorMessage} — will retry in {state.Interval}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user