fix(site-runtime): ConnectionQualityChanged(Good) only resets the tag-subscribe backoff while it is stretched — a flapping connection can no longer defer the retry it was meant to hasten
Review finding on ab08ecd3: each Good re-armed the watchdog from 'now', so Goods arriving faster than the floor starved the retry. Pinned by FlappingGood_DoesNotDeferTheRetry (12 Goods over 1.2 s; naive re-arm measured 1.43 s, guarded ~0.3 s).
Claude-Session: https://claude.ai/code/session_019ZxXT5sxZs5XrtdwYzJDPV
This commit is contained in:
@@ -1177,9 +1177,13 @@ public class InstanceActor : ReceiveActor
|
|||||||
// TagSubscribeRetryMaxIntervalMs — recovery must be prompt. Timer != null ⇔ the
|
// TagSubscribeRetryMaxIntervalMs — recovery must be prompt. Timer != null ⇔ the
|
||||||
// subscribe is unconfirmed (it is only nulled by a Success); if a request is in
|
// subscribe is unconfirmed (it is only nulled by a Success); if a request is in
|
||||||
// flight and young the re-armed tick just hits the guard and re-arms at the floor.
|
// flight and young the re-armed tick just hits the guard and re-arms at the floor.
|
||||||
|
// Only acts while the interval is actually stretched: once reset to the floor a
|
||||||
|
// further Good is a no-op, so a flapping connection cannot keep pushing the armed
|
||||||
|
// tick out and starve the retry it was meant to hasten.
|
||||||
if (qualityChanged.Quality == QualityCode.Good
|
if (qualityChanged.Quality == QualityCode.Good
|
||||||
&& _tagSubscribeStates.TryGetValue(qualityChanged.ConnectionName, out var subState)
|
&& _tagSubscribeStates.TryGetValue(qualityChanged.ConnectionName, out var subState)
|
||||||
&& subState.Timer != null)
|
&& subState.Timer != null
|
||||||
|
&& subState.Interval > TimeSpan.FromMilliseconds(_options.TagSubscribeRetryIntervalMs))
|
||||||
{
|
{
|
||||||
ResetTagSubscribeBackoff(subState);
|
ResetTagSubscribeBackoff(subState);
|
||||||
ArmTagSubscribeWatchdog(qualityChanged.ConnectionName, subState);
|
ArmTagSubscribeWatchdog(qualityChanged.ConnectionName, subState);
|
||||||
|
|||||||
@@ -1442,6 +1442,43 @@ public class InstanceActorTests : TestKit, IDisposable
|
|||||||
|
|
||||||
// --- Lever 1 (2026-08-28): Good quality resets the backoff ---
|
// --- Lever 1 (2026-08-28): Good quality resets the backoff ---
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void FlappingGood_DoesNotDeferTheRetry()
|
||||||
|
{
|
||||||
|
// A burst of Good events faster than the floor must not keep re-arming the watchdog
|
||||||
|
// from "now" — after the first reset the interval is at the floor and later Goods are
|
||||||
|
// no-ops, so the retry still lands within one floor interval of the first Good.
|
||||||
|
var options = new SiteRuntimeOptions
|
||||||
|
{
|
||||||
|
TagSubscribeRetryIntervalMs = 300,
|
||||||
|
TagSubscribeRetryMaxIntervalMs = 5000,
|
||||||
|
TagSubscribeInFlightCeilingMs = 60_000,
|
||||||
|
};
|
||||||
|
var (instance, dclProbe) = CreateInstanceWithDataSourcedAttribute(options);
|
||||||
|
|
||||||
|
var req = dclProbe.ExpectMsg<SubscribeTagsRequest>();
|
||||||
|
for (var i = 0; i < 3; i++)
|
||||||
|
{
|
||||||
|
instance.Tell(new SubscribeTagsResponse(req.CorrelationId, req.InstanceUniqueName,
|
||||||
|
false, "busy", DateTimeOffset.UtcNow), dclProbe.Ref);
|
||||||
|
req = dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(5));
|
||||||
|
}
|
||||||
|
instance.Tell(new SubscribeTagsResponse(req.CorrelationId, req.InstanceUniqueName,
|
||||||
|
false, "busy", DateTimeOffset.UtcNow), dclProbe.Ref);
|
||||||
|
|
||||||
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
// Twelve Goods over ~1.2 s — a naive re-arm-from-now on each would push the tick to ≥1.5 s.
|
||||||
|
for (var i = 0; i < 12; i++)
|
||||||
|
{
|
||||||
|
instance.Tell(new ConnectionQualityChanged("conn-1", QualityCode.Good, DateTimeOffset.UtcNow));
|
||||||
|
Thread.Sleep(100);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The probe already holds the retry (it landed ~300 ms after the first Good) by the time the burst ends.
|
||||||
|
dclProbe.ExpectMsg<SubscribeTagsRequest>(TimeSpan.FromSeconds(2));
|
||||||
|
Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(1400), $"retry took {sw.Elapsed} — flapping Good deferred it");
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ConnectionBackToGood_ResetsBackoff_AndRetriesPromptly()
|
public void ConnectionBackToGood_ResetsBackoff_AndRetriesPromptly()
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user