feat(site-runtime): tag-subscribe in-flight guard — one outstanding request per connection until TagSubscribeInFlightCeilingMs (Lever 1)

FailedSubscribeResponse_IsRetried and LostSubscribeResponse_IsRetried_AndSuccessStopsRetrying now pin the in-flight ceiling to the retry interval: a lost reply is re-sent only once the outstanding request is presumed lost, by design.

Claude-Session: https://claude.ai/code/session_019ZxXT5sxZs5XrtdwYzJDPV

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/InstanceActor.cs

@@ -178,14 +178,30 @@ public class InstanceActor : ReceiveActor
     // fan out to all of them — not just the last one registered.
     private readonly Dictionary<string, List<string>> _tagPathToAttributes = new();
 
-    // Tag-subscription retry state (S4/UA6): tag paths grouped by connection name
-    // (built once in SubscribeToDcl), the in-flight correlation-id → connection
-    // map, and a per-connection retry timer. Arming the timer on *send* closes both
-    // the failed-response and the lost-response gaps with one mechanism; the timer
-    // is cancelled when a Success reply arrives.
+    // Tag-subscription retry state (S4/UA6 + Lever 1 2026-08-28): tag paths grouped by
+    // connection name (built once in SubscribeToDcl), the in-flight correlation-id →
+    // connection map, and per-connection retry state. Arming the watchdog on *send*
+    // closes both the failed-response and the lost-response gaps with one mechanism;
+    // it is cancelled when a Success reply arrives. Lever 1 adds the in-flight guard
+    // (a tick while a young request is outstanding re-arms instead of re-sending) and
+    // exponential backoff (see NextTagSubscribeInterval) so a slow gateway is never
+    // handed a fresh N-tag subscribe on top of the one it is still processing.
     private readonly Dictionary<string, List<string>> _tagsByConnection = new();
     private readonly Dictionary<string, string> _pendingTagSubscribes = new();
-    private readonly Dictionary<string, ICancelable> _tagSubscribeRetryTimers = new();
+    private readonly Dictionary<string, TagSubscribeState> _tagSubscribeStates = new();
+
+    /// <summary>Per-connection tag-subscribe retry bookkeeping (Lever 1).</summary>
+    private sealed class TagSubscribeState
+    {
+        /// <summary>Correlation id of the outstanding request, null when none is in flight.</summary>
+        public string? InFlightCorrelationId;
+        /// <summary>When the outstanding request was sent (monotonic, for the ceiling check).</summary>
+        public long InFlightSentTicks;
+        /// <summary>Interval the NEXT retry will use; starts at the floor, grows on failure.</summary>
+        public TimeSpan Interval;
+        /// <summary>The armed watchdog for this connection.</summary>
+        public ICancelable? Timer;
+    }
 
     /// <summary>
     /// Initializes the instance actor with its configuration and dependencies.
@@ -388,10 +404,10 @@ public class InstanceActor : ReceiveActor
     /// <inheritdoc />
     protected override void PostStop()
     {
-        // Cancel any armed tag-subscribe retry timers so they cannot fire after stop.
-        foreach (var timer in _tagSubscribeRetryTimers.Values)
-            timer.Cancel();
-        _tagSubscribeRetryTimers.Clear();
+        // Cancel any armed tag-subscribe watchdogs so they cannot fire after stop.
+        foreach (var state in _tagSubscribeStates.Values)
+            state.Timer?.Cancel();
+        _tagSubscribeStates.Clear();
 
         // Final flush of any coalesced static-override writes so a graceful stop does not lose
         // the last <= flush-interval of writes (P4 pattern, mirrors NativeAlarmActor.PostStop).
@@ -1246,10 +1262,17 @@ public class InstanceActor : ReceiveActor
 
     /// <summary>
     /// Sends (or re-sends) the tag-subscribe request for one connection and arms a
-    /// per-connection retry timer (S4/UA6). Arming on send means the retry fires
-    /// whether the response is a failure OR is lost entirely; a Success reply
-    /// cancels it. A superseded in-flight correlation id is overwritten so only the
-    /// latest attempt's reply is honoured.
+    /// per-connection watchdog (S4/UA6). Arming on send means the retry fires whether
+    /// the response is a failure OR is lost entirely; a Success reply cancels it.
+    /// <para>
+    /// Lever 1 (2026-08-28): if a request for this connection is already in flight and
+    /// younger than <see cref="SiteRuntimeOptions.TagSubscribeInFlightCeilingMs"/>, no
+    /// new request is issued — the watchdog is simply re-armed at the current interval
+    /// and the outstanding request keeps its correlation id. This is the guard that
+    /// stops a slow gateway from receiving one new N-tag subscribe per interval. Past
+    /// the ceiling the reply is presumed lost: the old id is dropped (its late reply, if
+    /// any, is ignored as superseded), the interval backs off, and a fresh request goes out.
+    /// </para>
     /// </summary>
     private void SendTagSubscribe(string connectionName)
     {
@@ -1257,23 +1280,73 @@ public class InstanceActor : ReceiveActor
         if (!_tagsByConnection.TryGetValue(connectionName, out var tagPaths) || tagPaths.Count == 0)
             return;
 
+        var state = GetTagSubscribeState(connectionName);
+        var now = Environment.TickCount64;
+
+        if (state.InFlightCorrelationId != null)
+        {
+            var ageMs = now - state.InFlightSentTicks;
+            if (ageMs < _options.TagSubscribeInFlightCeilingMs)
+            {
+                _logger.LogDebug(
+                    "Instance {Instance} tag subscribe on connection {Connection} still in flight ({AgeMs}ms, corr={Corr}) — not re-sending",
+                    _instanceUniqueName, connectionName, ageMs, state.InFlightCorrelationId);
+                ArmTagSubscribeWatchdog(connectionName, state);
+                return;
+            }
+
+            _logger.LogWarning(
+                "Instance {Instance} tag subscribe on connection {Connection} unanswered for {AgeMs}ms (corr={Corr}) — presuming lost, re-sending",
+                _instanceUniqueName, connectionName, ageMs, state.InFlightCorrelationId);
+            _pendingTagSubscribes.Remove(state.InFlightCorrelationId);
+            BackOffTagSubscribe(state);
+        }
+
         var correlationId = Guid.NewGuid().ToString("N");
         _pendingTagSubscribes[correlationId] = connectionName;
+        state.InFlightCorrelationId = correlationId;
+        state.InFlightSentTicks = now;
 
         var request = new SubscribeTagsRequest(
             correlationId, _instanceUniqueName, connectionName, tagPaths, DateTimeOffset.UtcNow);
         _dclManager.Tell(request, Self);
         _logger.LogInformation(
-            "Instance {Instance} subscribing to {Count} tags on connection {Connection} (corr={Corr})",
-            _instanceUniqueName, tagPaths.Count, connectionName, correlationId);
+            "Instance {Instance} subscribing to {Count} tags on connection {Connection} (corr={Corr}, next retry in {Interval})",
+            _instanceUniqueName, tagPaths.Count, connectionName, correlationId, state.Interval);
 
-        if (_tagSubscribeRetryTimers.Remove(connectionName, out var existing))
-            existing.Cancel();
-        _tagSubscribeRetryTimers[connectionName] = Context.System.Scheduler.ScheduleTellOnceCancelable(
-            TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs), Self,
-            new RetryTagSubscribe(connectionName), Self);
+        ArmTagSubscribeWatchdog(connectionName, state);
     }
 
+    private TagSubscribeState GetTagSubscribeState(string connectionName)
+    {
+        if (!_tagSubscribeStates.TryGetValue(connectionName, out var state))
+        {
+            state = new TagSubscribeState { Interval = TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs) };
+            _tagSubscribeStates[connectionName] = state;
+        }
+        return state;
+    }
+
+    /// <summary>(Re)arms the per-connection watchdog at the state's current interval.</summary>
+    private void ArmTagSubscribeWatchdog(string connectionName, TagSubscribeState state)
+    {
+        state.Timer?.Cancel();
+        state.Timer = Context.System.Scheduler.ScheduleTellOnceCancelable(
+            state.Interval, Self, new RetryTagSubscribe(connectionName), Self);
+    }
+
+    /// <summary>Advances the backoff after a failed or presumed-lost attempt.</summary>
+    private void BackOffTagSubscribe(TagSubscribeState state) =>
+        state.Interval = NextTagSubscribeInterval(
+            state.Interval,
+            TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs),
+            TimeSpan.FromMilliseconds(_options.TagSubscribeRetryMaxIntervalMs),
+            Random.Shared.NextDouble());
+
+    /// <summary>Resets the backoff to its floor (Success, or connection back to Good).</summary>
+    private void ResetTagSubscribeBackoff(TagSubscribeState state) =>
+        state.Interval = TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs);
+
     /// <summary>
     /// Handles the DCL's tag-subscribe response: on Success cancels the connection's
     /// retry timer; on failure logs + emits a site event and leaves the armed timer
@@ -1286,8 +1359,14 @@ public class InstanceActor : ReceiveActor
 
         if (response.Success)
         {
-            if (_tagSubscribeRetryTimers.Remove(connectionName, out var timer))
-                timer.Cancel();
+            var state = GetTagSubscribeState(connectionName);
+            if (state.InFlightCorrelationId == response.CorrelationId)
+            {
+                state.Timer?.Cancel();
+                state.Timer = null;
+                state.InFlightCorrelationId = null;
+                ResetTagSubscribeBackoff(state);
+            }
             _logger.LogDebug(
                 "Instance {Instance} tag subscription confirmed on connection {Connection}",
                 _instanceUniqueName, connectionName);