Joseph Doherty
2d25c916b6
GalaxyDriver.ShutdownAsync was a no-op: the MX session, both owned gRPC channels, the EventPump, the alarm feed, the DeployWatcher, the probe watcher and the reconnect supervisor were released only by DisposeAsync, which the host never calls — one leaked gateway session per respawn, removal and /raw browse, and a watchdog Shutdown→Initialize that neither tore down nor rebuilt anything (review 2026-10-21 G-1, G-2, G-9, G-3). - ShutdownAsync and DisposeAsync share one ordered TeardownRuntimeAsync (supervisor first, then deploy/probe watchers, pump, alarm feed, session, channels); owned seams are nulled so the next InitializeAsync rebuilds; injected seams survive a shutdown and drop only on the final dispose. Lifecycle calls are serialised behind a gate. - A failed production connect releases its partial runtime (no orphaned client per watchdog retry) and lands Faulted before and after. - StartDeployWatcherAsync stops the previous watcher before starting a replacement, and now starts on a production instance's FIRST InitializeAsync (the old guard returned whenever discovery had not run here, which on a runtime instance is always). - ReconnectSupervisor releases each cycle's CancellationTokenSource when its loop exits and gains an awaitable StopAsync; PerPlatformProbeWatcher gains an awaitable StopAsync. - EventPump treats any exception under cancellation as a clean stop, so a teardown never wakes the supervisor. - Internal seams for tests: an IGalaxyDeployWatchSource ctor parameter and a session-open override forwarded to the owned GalaxyMxSession. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
93 lines
3.9 KiB
C#
93 lines
3.9 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
|
|
|
/// <summary>
|
|
/// Review 2026-10-21 G-3: <see cref="ReconnectSupervisor.ReportTransportFailure"/> allocated a
|
|
/// new <see cref="CancellationTokenSource"/> per recovery cycle and only the last one was ever
|
|
/// disposed. Each cycle now releases its own CTS when its loop exits, and
|
|
/// <see cref="ReconnectSupervisor.StopAsync"/> awaits the loop instead of blocking on it.
|
|
/// </summary>
|
|
public sealed class ReconnectSupervisorCycleCtsTests
|
|
{
|
|
private const int WaitMs = 10_000;
|
|
|
|
private static ReconnectOptions FastBackoff() =>
|
|
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
|
|
|
|
/// <summary>Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control).</summary>
|
|
[Fact]
|
|
public async Task Every_completed_recovery_cycle_releases_its_own_cts()
|
|
{
|
|
var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var hold = false;
|
|
using var sup = new ReconnectSupervisor(
|
|
reopen: async _ =>
|
|
{
|
|
if (hold)
|
|
{
|
|
reopenEntered.TrySetResult();
|
|
await holdReopen.Task;
|
|
}
|
|
},
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
sup.ReportTransportFailure(new IOException($"drop {i}"));
|
|
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
|
|
}
|
|
await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS");
|
|
sup.CyclesStarted.ShouldBe(3);
|
|
|
|
// Positive control: a cycle parked inside reopen still holds its CTS.
|
|
hold = true;
|
|
sup.ReportTransportFailure(new IOException("drop 3"));
|
|
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
|
|
sup.CyclesStarted.ShouldBe(4);
|
|
sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits");
|
|
|
|
holdReopen.TrySetResult();
|
|
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
|
|
await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit");
|
|
}
|
|
|
|
/// <summary>StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking.</summary>
|
|
[Fact]
|
|
public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts()
|
|
{
|
|
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var sup = new ReconnectSupervisor(
|
|
reopen: async ct =>
|
|
{
|
|
reopenEntered.TrySetResult();
|
|
await Task.Delay(Timeout.Infinite, ct);
|
|
},
|
|
replay: _ => Task.CompletedTask,
|
|
options: FastBackoff());
|
|
|
|
sup.ReportTransportFailure(new IOException("drop"));
|
|
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
|
|
|
|
var stop = sup.StopAsync();
|
|
(await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return");
|
|
sup.CyclesReleased.ShouldBe(1);
|
|
await Should.NotThrowAsync(sup.StopAsync); // idempotent
|
|
Should.NotThrow(sup.Dispose);
|
|
}
|
|
|
|
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
|
{
|
|
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
|
while (!condition())
|
|
{
|
|
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
|
await Task.Delay(10);
|
|
}
|
|
}
|
|
}
|