fix(galaxy): #865 review round 4 - replay fails on a dead stream, backoff-aware wait
- ReplayAsync fails when its restarted pump's stream has already ended, so the supervisor backs off and stays Degraded instead of going Healthy over a dead stream (whose next advise re-report restarted at attempt 1 with no backoff - a tight reconnect loop). - ReconnectSupervisor.AttemptInProgress (reopen through replay); advises wait for a stale pump's replacement only while an attempt is running, not for a whole backoff (the state still reads Reopening/Replaying there). - ReconnectSupervisor: _disposed is set under the state lock and re-checked inside ReportTransportFailure, so a late report cannot start a recovery loop that StopAsync already missed. - EventPump.HasEnded is a flag set before StreamRequested is released, and a stream that ends/faults on its first MoveNext is marked ended first. - SubscriptionRegistry fast path: a miss now falls through to the locked path (lookup + session + window check atomically) instead of a lock-free re-lookup. - Tests: replay over an immediately-ending stream fails; AttemptInProgress across attempt/backoff; ReportTransportFailure after stop; the reopen-window test uses a generous wait bound. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -51,6 +51,11 @@ public sealed class ProbeInitialValueTests
|
||||
/// <summary>Initial values pushed while no stream was attached — lost, as on the real gateway.</summary>
|
||||
public int DroppedInitialValues;
|
||||
|
||||
private int _streams;
|
||||
|
||||
/// <summary>Streams beyond this count end immediately after attaching (a gateway refusing/ending them).</summary>
|
||||
public int StreamsThatStayOpen = int.MaxValue;
|
||||
|
||||
/// <summary>The ScanState value the next advise reports.</summary>
|
||||
public volatile bool ScanStateRunning = true;
|
||||
|
||||
@@ -90,6 +95,11 @@ public sealed class ProbeInitialValueTests
|
||||
Calls.Add("stream");
|
||||
_attached = mine;
|
||||
}
|
||||
if (Interlocked.Increment(ref _streams) > StreamsThatStayOpen)
|
||||
{
|
||||
lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; }
|
||||
yield break; // the gateway completed this stream at once
|
||||
}
|
||||
try
|
||||
{
|
||||
await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
@@ -208,6 +218,9 @@ public sealed class ProbeInitialValueTests
|
||||
platformSource: new FixedPlatformSource(Platforms))
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
// Generous: the window advise must still be waiting when the replay below starts the pump, even
|
||||
// on a stalled agent; the wait ends the moment that pump exists.
|
||||
StreamRequestWait = TimeSpan.FromSeconds(20),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await TestWait.UntilAsync(
|
||||
@@ -315,6 +328,27 @@ public sealed class ProbeInitialValueTests
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A replay whose restarted pump's stream ends at once must FAIL (the supervisor then backs off and
|
||||
/// stays Degraded) rather than succeed onto a dead stream — succeeding went Healthy, and the next
|
||||
/// advise's re-report started a new cycle with no backoff: a tight reconnect loop. Positive control:
|
||||
/// the first replay, whose stream stays open, succeeds.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Replay_fails_when_the_restarted_stream_ends_at_once()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber { StreamsThatStayOpen = 2 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // stream 1
|
||||
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None); // stream 2 stays open — succeeds
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => driver.InvokeReplayForTestAsync(CancellationToken.None), // stream 3 ends at once
|
||||
"a replay must not report success over a dead event stream");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A probe sync on a driver whose pump was never started (no platform source, e.g. the
|
||||
/// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just
|
||||
|
||||
+50
@@ -15,6 +15,56 @@ public sealed class ReconnectSupervisorTests
|
||||
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
|
||||
MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
|
||||
|
||||
/// <summary>
|
||||
/// #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff
|
||||
/// between attempts — when the state still reads Reopening. Advises wait for a replay only while one is
|
||||
/// actually running, not for a whole backoff.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff()
|
||||
{
|
||||
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var reopens = 0;
|
||||
using var sup = new ReconnectSupervisor(
|
||||
reopen: async _ =>
|
||||
{
|
||||
if (Interlocked.Increment(ref reopens) == 1)
|
||||
{
|
||||
reopenEntered.TrySetResult();
|
||||
await releaseReopen.Task;
|
||||
throw new IOException("gateway still down");
|
||||
}
|
||||
},
|
||||
replay: _ => Task.CompletedTask,
|
||||
backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff
|
||||
|
||||
sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt");
|
||||
sup.ReportTransportFailure(new IOException("drop"));
|
||||
await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start");
|
||||
sup.AttemptInProgress.ShouldBeTrue("a reopen is running");
|
||||
|
||||
releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min
|
||||
await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag");
|
||||
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists");
|
||||
}
|
||||
|
||||
/// <summary>#865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE).</summary>
|
||||
[Fact]
|
||||
public async Task ReportTransportFailure_after_stop_never_starts_a_loop()
|
||||
{
|
||||
var reopens = 0;
|
||||
var sup = new ReconnectSupervisor(
|
||||
reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; },
|
||||
replay: _ => Task.CompletedTask,
|
||||
options: FastBackoff());
|
||||
await sup.StopAsync();
|
||||
|
||||
Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("late")));
|
||||
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
|
||||
Volatile.Read(ref reopens).ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the supervisor starts in a healthy state with no errors.</summary>
|
||||
[Fact]
|
||||
public void InitialState_IsHealthy()
|
||||
|
||||
Reference in New Issue
Block a user