feat(site-runtime): ConnectionQualityChanged(Good) resets tag-subscribe backoff to the floor (Lever 1); lock in superseded-reply drop

Task 4 review fix-ups: RepeatedFailures gaps[1] lower bound 600→500 (two compounding jitter draws bottom out at 512 ms); SubscribeToDcl also clears _tagPathToAttributes and its prune is documented as defensive (PreStart is the only caller).
This commit is contained in:
Joseph Doherty
2026-08-28 14:50:04 -04:00
parent b6b401c879
commit ab08ecd3d3
2 changed files with 81 additions and 3 deletions
@@ -186,8 +186,8 @@ public class InstanceActor : ReceiveActor
// (a tick while a young request is outstanding re-arms instead of re-sending) and
// exponential backoff (see NextTagSubscribeInterval) so a slow gateway is never
// handed a fresh N-tag subscribe on top of the one it is still processing.
// A re-entry into SubscribeToDcl rebuilds all three together: the retry state is
// pruned (timers cancelled, pending rows dropped) before the new grouping is built.
// SubscribeToDcl has a single caller today (PreStart); the prune at its top is defensive, so a
// future second call site rebuilds the retry state and both tag maps together instead of stranding them.
private readonly Dictionary<string, List<string>> _tagsByConnection = new();
private readonly Dictionary<string, string> _pendingTagSubscribes = new();
private readonly Dictionary<string, TagSubscribeState> _tagSubscribeStates = new();
@@ -1171,6 +1171,20 @@ public class InstanceActor : ReceiveActor
_logger.LogWarning("Connection {Connection} quality changed to {Quality} for instance {Instance}",
qualityChanged.ConnectionName, qualityChanged.Quality, _instanceUniqueName);
// Lever 1 (2026-08-28): a connection returning to Good is the Instance Actor's
// only restore signal (the DCL re-subscribes transparently). If our own subscribe
// on it is still unconfirmed, retry at the floor now rather than in up to
// TagSubscribeRetryMaxIntervalMs — recovery must be prompt. Timer != null ⇔ the
// subscribe is unconfirmed (it is only nulled by a Success); if a request is in
// flight and young the re-armed tick just hits the guard and re-arms at the floor.
if (qualityChanged.Quality == QualityCode.Good
&& _tagSubscribeStates.TryGetValue(qualityChanged.ConnectionName, out var subState)
&& subState.Timer != null)
{
ResetTagSubscribeBackoff(subState);
ArmTagSubscribeWatchdog(qualityChanged.ConnectionName, subState);
}
if (_configuration == null) return;
// Mark all attributes bound to this connection with the new quality
@@ -1208,6 +1222,7 @@ public class InstanceActor : ReceiveActor
{
if (_dclManager == null || _configuration == null) return;
// Defensive — PreStart is the only caller today.
// Lever 1: prune retry state from a previous build — cancels any armed watchdog and
// drops its pending row so a re-entry can never strand an in-flight slot.
foreach (var state in _tagSubscribeStates.Values)
@@ -1221,6 +1236,7 @@ public class InstanceActor : ReceiveActor
// Group attributes by their bound connection name into the persistent
// _tagsByConnection field so a retry can re-send the exact same request.
_tagsByConnection.Clear();
_tagPathToAttributes.Clear();
foreach (var attr in _configuration.Attributes)
{
if (string.IsNullOrEmpty(attr.DataSourceReference) ||
@@ -1410,7 +1410,8 @@ public class InstanceActorTests : TestKit, IDisposable
// Expected nominal gaps: 400, 800(cap), 800(cap) — each ±20 % jitter and scheduler slack.
Assert.InRange(gaps[0].TotalMilliseconds, 300, 560);
Assert.InRange(gaps[1].TotalMilliseconds, 600, 1100);
// gaps[1] compounds two jitter draws: 2×(2×200×0.8)×0.8 = 512 ms minimum.
Assert.InRange(gaps[1].TotalMilliseconds, 500, 1100);
Assert.InRange(gaps[2].TotalMilliseconds, 600, 1100);
// Success resets: no further request follows a confirmed subscription.
@@ -1438,4 +1439,65 @@ public class InstanceActorTests : TestKit, IDisposable
var retry = dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(3));
Assert.NotEqual(first.CorrelationId, retry.CorrelationId);
}
// --- Lever 1 (2026-08-28): Good quality resets the backoff ---
[Fact]
public void ConnectionBackToGood_ResetsBackoff_AndRetriesPromptly()
{
var options = new SiteRuntimeOptions
{
TagSubscribeRetryIntervalMs = 200,
TagSubscribeRetryMaxIntervalMs = 5000,
TagSubscribeInFlightCeilingMs = 60_000,
};
var (instance, dclProbe) = CreateInstanceWithDataSourcedAttribute(options);
// Drive the interval up: 3 failures → 400, 800, 1600 (nominal).
var req = dclProbe.ExpectMsg<SubscribeTagsRequest>();
for (var i = 0; i < 3; i++)
{
instance.Tell(new SubscribeTagsResponse(req.CorrelationId, req.InstanceUniqueName,
false, "busy", DateTimeOffset.UtcNow), dclProbe.Ref);
req = dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(5));
}
// One more failure would nominally wait ~3.2 s. Instead the connection recovers:
instance.Tell(new SubscribeTagsResponse(req.CorrelationId, req.InstanceUniqueName,
false, "busy", DateTimeOffset.UtcNow), dclProbe.Ref);
instance.Tell(new ConnectionQualityChanged("conn-1", QualityCode.Good, DateTimeOffset.UtcNow));
var sw = System.Diagnostics.Stopwatch.StartNew();
dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(2));
Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(1000), $"retry took {sw.Elapsed} — backoff not reset");
}
[Fact]
public void LateReplyForFailedAndReissuedRequest_IsIgnored()
{
var options = new SiteRuntimeOptions
{
TagSubscribeRetryIntervalMs = 200,
TagSubscribeRetryMaxIntervalMs = 200,
// Short ceiling on purpose: the second request stays in flight (nothing ever
// answers it), so the in-flight guard is what re-sends. With a 60 s ceiling the
// guard would hold for 60 s and the test could not tell "late reply ignored"
// apart from "late reply wrongly cleared the slot" — both look like silence.
TagSubscribeInFlightCeilingMs = 500,
};
var (instance, dclProbe) = CreateInstanceWithDataSourcedAttribute(options);
var first = dclProbe.ExpectMsg<SubscribeTagsRequest>();
instance.Tell(new SubscribeTagsResponse(first.CorrelationId, first.InstanceUniqueName,
false, "busy", DateTimeOffset.UtcNow), dclProbe.Ref);
var second = dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(3));
// Duplicate/late Success for the FIRST id: already removed from the pending map → ignored,
// so the watchdog guarding the SECOND is untouched and still re-sends once its
// in-flight ceiling expires. Had the late reply been honoured it would have cancelled
// that watchdog and cleared the slot, and no third request would ever be sent.
instance.Tell(new SubscribeTagsResponse(first.CorrelationId, first.InstanceUniqueName,
true, null, DateTimeOffset.UtcNow), dclProbe.Ref);
var third = dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(3));
Assert.NotEqual(second.CorrelationId, third.CorrelationId);
}
}