8653960ea3
- ReplayAsync fails when its restarted pump's stream has already ended, so the supervisor backs off and stays Degraded instead of going Healthy over a dead stream (whose next advise re-report restarted at attempt 1 with no backoff - a tight reconnect loop). - ReconnectSupervisor.AttemptInProgress (reopen through replay); advises wait for a stale pump's replacement only while an attempt is running, not for a whole backoff (the state still reads Reopening/Replaying there). - ReconnectSupervisor: _disposed is set under the state lock and re-checked inside ReportTransportFailure, so a late report cannot start a recovery loop that StopAsync already missed. - EventPump.HasEnded is a flag set before StreamRequested is released, and a stream that ends/faults on its first MoveNext is marked ended first. - SubscriptionRegistry fast path: a miss now falls through to the locked path (lookup + session + window check atomically) instead of a lock-free re-lookup. - Tests: replay over an immediately-ending stream fails; AttemptInProgress across attempt/backoff; ReportTransportFailure after stop; the reopen-window test uses a generous wait bound. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
242 lines
11 KiB
C#
242 lines
11 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
|
|
|
/// <summary>
|
|
/// Tests for <see cref="ReconnectSupervisor"/>'s state machine + backoff. Each
|
|
/// scenario drives the supervisor with controllable reopen/replay callbacks and
|
|
/// observes the resulting state transitions.
|
|
/// </summary>
|
|
public sealed class ReconnectSupervisorTests
|
|
{
|
|
private static ReconnectOptions FastBackoff() =>
|
|
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
|
|
MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
|
|
|
|
/// <summary>
|
|
/// #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff
|
|
/// between attempts — when the state still reads Reopening. Advises wait for a replay only while one is
|
|
/// actually running, not for a whole backoff.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff()
|
|
{
|
|
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var reopens = 0;
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: async _ =>
|
|
{
|
|
if (Interlocked.Increment(ref reopens) == 1)
|
|
{
|
|
reopenEntered.TrySetResult();
|
|
await releaseReopen.Task;
|
|
throw new IOException("gateway still down");
|
|
}
|
|
},
|
|
replay: _ => Task.CompletedTask,
|
|
backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff
|
|
|
|
sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt");
|
|
sup.ReportTransportFailure(new IOException("drop"));
|
|
await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start");
|
|
sup.AttemptInProgress.ShouldBeTrue("a reopen is running");
|
|
|
|
releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min
|
|
await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag");
|
|
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists");
|
|
}
|
|
|
|
/// <summary>#865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE).</summary>
|
|
[Fact]
|
|
public async Task ReportTransportFailure_after_stop_never_starts_a_loop()
|
|
{
|
|
var reopens = 0;
|
|
var sup = new ReconnectSupervisor(
|
|
reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; },
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
await sup.StopAsync();
|
|
|
|
Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("late")));
|
|
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
|
|
Volatile.Read(ref reopens).ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>Verifies that the supervisor starts in a healthy state with no errors.</summary>
|
|
[Fact]
|
|
public void InitialState_IsHealthy()
|
|
{
|
|
using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
|
|
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
|
|
sup.IsDegraded.ShouldBeFalse();
|
|
sup.LastError.ShouldBeNull();
|
|
}
|
|
|
|
/// <summary>Verifies that reporting a transport failure drives the supervisor through reopen and replay cycles back to healthy.</summary>
|
|
[Fact]
|
|
public async Task ReportTransportFailure_DrivesThroughReopenReplay_BackToHealthy()
|
|
{
|
|
var transitions = new List<StateTransition>();
|
|
var lockObj = new object();
|
|
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: _ => Task.CompletedTask,
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
|
|
sup.StateChanged += (_, t) => { lock (lockObj) transitions.Add(t); };
|
|
|
|
sup.ReportTransportFailure(new IOException("transport drop"));
|
|
await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");
|
|
|
|
// Expected sequence: Healthy → TransportLost → Reopening → Replaying → Healthy.
|
|
IReadOnlyList<StateTransition> snapshot;
|
|
lock (lockObj) snapshot = [.. transitions];
|
|
var states = snapshot.Select(t => t.Next).ToArray();
|
|
|
|
states.ShouldContain(ReconnectSupervisor.State.TransportLost);
|
|
states.ShouldContain(ReconnectSupervisor.State.Reopening);
|
|
states.ShouldContain(ReconnectSupervisor.State.Replaying);
|
|
states[^1].ShouldBe(ReconnectSupervisor.State.Healthy);
|
|
sup.IsDegraded.ShouldBeFalse();
|
|
}
|
|
|
|
/// <summary>Verifies that reopen failures trigger retries and the supervisor stays in reopening state between attempts.</summary>
|
|
[Fact]
|
|
public async Task ReopenFailure_RetriesUntilSuccess_StaysInReopeningBetweenAttempts()
|
|
{
|
|
var attempts = 0;
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: _ => { attempts++; return attempts < 3 ? Task.FromException(new IOException("nope")) : Task.CompletedTask; },
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
|
|
sup.ReportTransportFailure(new IOException("kick off"));
|
|
await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");
|
|
|
|
attempts.ShouldBe(3);
|
|
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
|
|
sup.LastError.ShouldBeNull(); // cleared on Healthy transition
|
|
}
|
|
|
|
/// <summary>Verifies that replay failures trigger a retry of the entire reopen-replay cycle.</summary>
|
|
[Fact]
|
|
public async Task ReplayFailure_RetriesEntireCycle()
|
|
{
|
|
var reopens = 0;
|
|
var replays = 0;
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: _ => { reopens++; return Task.CompletedTask; },
|
|
replay: _ => { replays++; return replays < 2 ? Task.FromException(new IOException("replay nope")) : Task.CompletedTask; },
|
|
options: FastBackoff());
|
|
|
|
sup.ReportTransportFailure(new IOException("kick off"));
|
|
await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");
|
|
|
|
// First cycle: reopen succeeds, replay fails. Second cycle: both succeed.
|
|
reopens.ShouldBe(2);
|
|
replays.ShouldBe(2);
|
|
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
|
|
}
|
|
|
|
/// <summary>Verifies that repeated failure reports during recovery do not spawn parallel recovery loops.</summary>
|
|
[Fact]
|
|
public async Task RepeatedFailureReports_DuringRecovery_DoNotSpawnParallelLoops()
|
|
{
|
|
var attempts = 0;
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: async ct =>
|
|
{
|
|
attempts++;
|
|
await Task.Delay(50, ct).ConfigureAwait(false);
|
|
},
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
|
|
sup.ReportTransportFailure(new IOException("first"));
|
|
// Fire several more reports while reopen is in flight.
|
|
for (var i = 0; i < 5; i++)
|
|
{
|
|
sup.ReportTransportFailure(new IOException($"rapid-{i}"));
|
|
}
|
|
|
|
await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");
|
|
|
|
// One Reopen call regardless of how many failures arrived during recovery.
|
|
attempts.ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>Verifies that the last error reflects the most recent failure cause from recovery attempts.</summary>
|
|
[Fact]
|
|
public async Task LastError_ReflectsMostRecentFailureCause()
|
|
{
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: _ => Task.FromException(new IOException("reopen broke")),
|
|
replay: _ => Task.CompletedTask,
|
|
options: new ReconnectOptions(
|
|
InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
|
|
MaxBackoffOverride: TimeSpan.FromMilliseconds(10)));
|
|
|
|
sup.ReportTransportFailure(new IOException("initial"));
|
|
|
|
// Wait for the loop's first failed reopen to land rather than sleeping a fixed 100 ms: on a
|
|
// loaded CI runner the recovery loop had not been scheduled yet, so LastError still held the
|
|
// reported cause ("initial") — #606. A presence wait returns the instant the condition holds.
|
|
await TestWait.UntilAsync(
|
|
() => sup.LastError?.Contains("reopen broke", StringComparison.OrdinalIgnoreCase) == true,
|
|
"the loop's failed reopen must replace the reported cause",
|
|
() => $"state={sup.CurrentState} lastError={sup.LastError ?? "<none>"}");
|
|
|
|
sup.LastError.ShouldNotBeNull();
|
|
sup.LastError.ShouldContain("reopen broke"); // updates from the loop's failed reopen attempts
|
|
}
|
|
|
|
/// <summary>Verifies that disposing the supervisor cancels a running recovery loop cleanly.</summary>
|
|
[Fact]
|
|
public async Task Dispose_CancelsRunningRecoveryLoop_Cleanly()
|
|
{
|
|
var cancelled = false;
|
|
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var sup = new ReconnectSupervisor(
|
|
reopen: async ct =>
|
|
{
|
|
reopenEntered.TrySetResult();
|
|
try { await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); }
|
|
catch (OperationCanceledException) { cancelled = true; throw; }
|
|
},
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
|
|
sup.ReportTransportFailure(new IOException("kick off"));
|
|
// Wait for the loop to be INSIDE the long reopen (#858: a fixed 50 ms sleep raced the loop's
|
|
// scheduling under load, and a Dispose before reopen starts never reaches the catch).
|
|
await TestWait.ForAsync(reopenEntered.Task, "the recovery loop must enter reopen");
|
|
Should.NotThrow(() => sup.Dispose());
|
|
cancelled.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>Verifies that reporting a transport failure after dispose throws ObjectDisposedException.</summary>
|
|
[Fact]
|
|
public void ReportTransportFailure_AfterDispose_Throws()
|
|
{
|
|
var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
|
|
sup.Dispose();
|
|
Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("x")));
|
|
}
|
|
|
|
/// <summary>Verifies that waiting for healthy state returns immediately when already healthy.</summary>
|
|
[Fact]
|
|
public async Task WaitForHealthy_ReturnsImmediately_WhenAlreadyHealthy()
|
|
{
|
|
using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
|
|
// No failure reported — should be Healthy from the start, so the wait completes synchronously
|
|
// (deterministic, unlike a wall-clock bound that a descheduled thread can overrun — #858).
|
|
var wait = sup.WaitForHealthyAsync(CancellationToken.None);
|
|
wait.IsCompletedSuccessfully.ShouldBeTrue("an already-Healthy supervisor must not wait at all");
|
|
await wait;
|
|
}
|
|
}
|