using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
///
/// Review 02 H2: the Reconnecting retry timer fired every reconnectInterval whether or not the
/// previous attempt was still running — _initGeneration only discarded the older attempt's RESULT. With an
/// Initialize bound longer than the cadence (Tier C's 15 s vs. the 10 s timer, or the test harness's unbounded
/// invoker) attempt N+1's ShutdownAsync ran while attempt N's InitializeAsync was still inside the
/// driver: the dispose-before-overwrite race the drivers' rule file forbids, once per overlap, for as long as
/// the device stayed down. The retry is now single-flight.
///
public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActorTestBase
{
[Fact]
public void Shutdown_and_Initialize_never_overlap_while_the_init_outlasts_the_retry_cadence()
{
// A 400 ms failing init against a 50 ms retry cadence: without the gate the timer starts a new
// Shutdown+Initialize pair every 50 ms on top of the one still running (eight-deep overlap).
var driver = new InterleavingDriver(initDuration: TimeSpan.FromMilliseconds(400), failUntilAttempt: 4);
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
// Positive control: the gate skips ticks, it does not stop retrying — the 4th attempt succeeds and the
// actor connects.
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
driver.Attempts.ShouldBeGreaterThanOrEqualTo(4);
driver.Overlaps.ShouldBe(0,
$"ShutdownAsync entered while an InitializeAsync was still running. Interleaving: {driver.Interleaving}");
}
[Fact]
public void A_pre_reconnect_teardown_that_hangs_is_abandoned_at_the_bound_and_the_init_still_runs()
{
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 2)
{
// Only the SECOND shutdown (the reconnect teardown) hangs; the first attempt has none.
ShutdownBehavior = _ => never.Task,
};
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
shutdownTimeout: TimeSpan.FromMilliseconds(200)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
never.Task.IsCompleted.ShouldBeFalse("the teardown is still hung — the attempt did not wait for it");
driver.Attempts.ShouldBe(2);
}
///
/// Fix round 1: the single-flight gate only cleared on the attempt's RESULT, and the invoker's timeout is
/// token-only — a driver whose InitializeAsync ignores its token wedged the reconnect loop forever. The
/// attempt is now bounded pessimistically (initializeTimeout); an overrun is abandoned, reported
/// as a failure, and the next tick retries.
///
[Fact]
public void A_token_ignoring_never_completing_init_is_abandoned_and_a_later_attempt_connects()
{
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 1)
{
// Attempt 1 hangs forever and ignores cancellation; attempt 2 completes.
HangOnAttempt = 1,
};
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
initializeTimeout: TimeSpan.FromMilliseconds(300)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
driver.Attempts.ShouldBe(2);
driver.HungAttemptStillRunning.ShouldBeTrue("positive control: the first attempt really was wedged");
}
/// Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.
private sealed class InterleavingDriver : IDriver
{
private readonly TimeSpan _initDuration;
private readonly int _failUntilAttempt;
private readonly Lock _gate = new();
private readonly List _events = new();
private int _attempts;
private int _initsInProgress;
private int _overlaps;
public InterleavingDriver(TimeSpan initDuration, int failUntilAttempt)
{
_initDuration = initDuration;
_failUntilAttempt = failUntilAttempt;
}
public Func? ShutdownBehavior { get; init; }
/// The 1-based attempt that never completes and ignores its token (0 = none).
public int HangOnAttempt { get; init; }
private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool HungAttemptStillRunning => !_never.Task.IsCompleted && Volatile.Read(ref _initsInProgress) > 0;
public int Attempts => Volatile.Read(ref _attempts);
public int Overlaps => Volatile.Read(ref _overlaps);
public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }
public string DriverInstanceId => "interleaving-1";
public string DriverType => "Interleaving";
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
{
var attempt = Interlocked.Increment(ref _attempts);
Interlocked.Increment(ref _initsInProgress);
Record($"init{attempt}>");
try
{
if (attempt == HangOnAttempt) await _never.Task;
if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
}
finally
{
Record($" Task.CompletedTask;
public async Task ShutdownAsync(CancellationToken cancellationToken)
{
if (Volatile.Read(ref _initsInProgress) > 0) Interlocked.Increment(ref _overlaps);
Record("shutdown>");
try
{
if (ShutdownBehavior is { } behavior) await behavior(cancellationToken);
}
finally
{
Record(" new(DriverState.Healthy, DateTime.UtcNow, null);
private void Record(string e) { lock (_gate) _events.Add(e); }
}
}