using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
///
/// Lane F fix round 1 (review of #735): the probe watcher is seeded at InitializeAsync
/// and re-seeded on rediscovery (the runtime never calls DiscoverAsync on a live driver);
/// a SubscribeAsync landing mid-teardown cannot create a pump on the dying subscriber; a
/// gateway close that never answers cannot park ShutdownAsync; a re-subscribe that
/// straddles a teardown never rebinds into the next runtime's registry.
///
public sealed class GalaxyDriverFixRound1Tests
{
internal const int WaitMs = 10_000;
internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("FixRound1"),
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
new GalaxyReconnectOptions());
// ───────────────────────────── fakes ─────────────────────────────
private sealed class ScriptedPlatformSource : IGalaxyPlatformSource
{
private int _calls;
public int Calls => Volatile.Read(ref _calls);
public Func> Script { get; init; } = _ => ["Plat01"];
public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
=> Task.FromResult(Script(Interlocked.Increment(ref _calls)));
}
internal sealed class ParkableSubscriber : IGalaxySubscriber
{
private readonly Channel _events = Channel.CreateUnbounded();
private int _nextHandle;
private int _subscribeCalls;
public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
public List Subscribed { get; } = [];
public int SubscribeCalls => Volatile.Read(ref _subscribeCalls);
/// When set, UnsubscribeBulk parks until .
public bool ParkUnsubscribe { get; set; }
public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult();
/// When > 0, that SubscribeBulk call (1-based) parks until .
public int ParkSubscribeCall { get; set; }
public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult();
public async Task> SubscribeBulkAsync(
IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
var call = Interlocked.Increment(ref _subscribeCalls);
if (call == ParkSubscribeCall)
{
SubscribeEntered.TrySetResult();
await _releaseSubscribe.Task.WaitAsync(cancellationToken);
}
var results = new List();
lock (Subscribed)
{
foreach (var r in fullReferences)
{
var h = Interlocked.Increment(ref _nextHandle);
Subscribed.Add(r);
HandleByRef[r] = h;
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
}
}
return results;
}
/// Every handle passed to UnsubscribeBulk, in call order.
public List Unsubscribed { get; } = [];
public async Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken)
{
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
if (ParkUnsubscribe)
{
UnsubscribeEntered.TrySetResult();
await _releaseUnsubscribe.Task.WaitAsync(cancellationToken);
}
}
public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { BoolValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
});
}
internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource
{
public Channel Events { get; } = Channel.CreateUnbounded();
public int Opened;
public async IAsyncEnumerable WatchAsync(
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
{
Interlocked.Increment(ref Opened);
await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent
{
Sequence = sequence,
TimeOfLastDeployPresent = true,
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
});
}
internal static async Task WaitUntilAsync(Func condition, string because)
{
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
await Task.Delay(10);
}
}
internal static HostState? StateOf(GalaxyDriver driver, string host) =>
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
// ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ─────────────────────────────
/// A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses.
[Fact]
public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync()
{
var sub = new ParkableSubscriber();
var platforms = new ScriptedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
await driver.InitializeAsync("{}", CancellationToken.None);
// The seed runs in the background (fix round 2, N1) — init does not wait on it.
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the probe watcher must be seeded after init");
platforms.Calls.ShouldBe(1);
lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState");
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
"a ScanState event must surface the seeded platform on GetHostStatuses");
}
/// A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds.
[Fact]
public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds()
{
var sub = new ParkableSubscriber();
var deploys = new ChannelDeployWatchSource();
var platforms = new ScriptedPlatformSource
{
Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"],
};
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
var raised = 0;
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
await driver.InitializeAsync("{}", CancellationToken.None);
driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init");
await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run");
sub.Subscribed.ShouldBeEmpty();
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed)
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy
await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery");
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
"the rediscovery must re-seed the probe set");
platforms.Calls.ShouldBe(2);
sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true);
}
private sealed class ParkingPlatformSource : IGalaxyPlatformSource
{
private int _calls;
public int Calls => Volatile.Read(ref _calls);
public int ParkCall { get; init; }
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _release.TrySetResult();
public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
{
var call = Interlocked.Increment(ref _calls);
if (call == ParkCall)
{
Entered.TrySetResult();
await _release.Task.WaitAsync(cancellationToken);
return ["Plat01"]; // the pre-deploy view this pass fetched
}
return call == 1 ? [] : ["Plat01", "Eng02"];
}
}
///
/// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts
/// after it — otherwise the probe set would stay on the stale view until the next deploy.
///
[Fact]
public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass()
{
var sub = new ParkableSubscriber();
var deploys = new ChannelDeployWatchSource();
var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
var raised = 0;
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
try
{
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
(await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task,
"the first rediscovery pass must be parked mid-fetch");
await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero));
await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery");
}
finally
{
platforms.Release();
}
await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed");
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
"the final probe set must reflect the latest deploy, not the parked pass's stale view");
}
/// The production path (owned runtime) queries the platform source at init, with no DiscoverAsync.
[Fact]
public async Task Production_path_queries_the_platform_source_at_init()
{
var platforms = new ScriptedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms)
{
SessionOpenOverrideForTests = _ => Task.CompletedTask,
};
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init");
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
}
// ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ─────────────────────────────
///
/// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before
/// the subscriber is released) is refused and leaves no pump; before the shutdown the same
/// call works (positive control).
///
[Fact]
public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump()
{
var sub = new ParkableSubscriber();
var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the (background) seed must give the probe watcher its handle first");
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
sub.ParkUnsubscribe = true;
var shutdown = driver.ShutdownAsync(CancellationToken.None);
var subscribeCallsBefore = sub.SubscribeCalls;
try
{
(await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task,
"the teardown must be parked inside the probe watcher's unsubscribe");
shutdown.IsCompleted.ShouldBeFalse();
// The pump is still live here (the teardown takes it after the probe watcher) — reusing it
// is exactly the race: the call must be refused, not served on the dying subscriber.
driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken");
await Should.ThrowAsync(
() => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None));
sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway");
}
finally
{
// Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs.
sub.ReleaseUnsubscribe();
}
await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown");
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
await driver.InitializeAsync("{}", CancellationToken.None);
driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
}
// ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
/// A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline.
[Fact]
public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers()
{
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource())
{
SessionOpenOverrideForTests = _ => Task.CompletedTask,
SessionCloseDeadline = TimeSpan.FromMilliseconds(300),
};
await driver.InitializeAsync("{}", CancellationToken.None);
var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var closeToken = CancellationToken.None;
driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct =>
{
closeToken = ct;
closeEntered.TrySetResult();
await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway
};
var sw = System.Diagnostics.Stopwatch.StartNew();
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
sw.Stop();
closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted");
closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline");
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s");
driver.OwnedMxSessionForTests.ShouldBeNull();
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
}
// ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ─────────────────────────────
///
/// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared
/// registry; the same replay without a teardown rebinds (positive control).
///
[Fact]
public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind()
{
// Positive control first.
{
var sub = new ParkableSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
await driver.InvokeReplayForTestAsync(CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription");
sub.SubscribeCalls.ShouldBe(2);
}
{
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
var replay = driver.InvokeReplayForTestAsync(CancellationToken.None);
(await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task,
"the replay must be parked inside its SubscribeBulk");
await driver.ShutdownAsync(CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
sub.ReleaseSubscribe();
await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles");
}
}
}