using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
///
/// #862 — with Reconnect.ReplayOnSessionLost=false a gateway reconnect keeps the driver's
/// subscriptions but the recreated session carries none of their advises. That used to be invisible:
/// every node kept its last value with Good quality and health went back to Healthy. Now each tag of
/// each stranded subscription is published BadNoCommunication and the driver reports Degraded
/// until those subscriptions are unsubscribed (or the runtime is torn down).
///
public sealed class StrandedSubscriptionTests
{
private const uint BadNoCommunication = 0x80310000u;
private static GalaxyDriverOptions Opts(bool replay) => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("OtOpcUa-A"),
new GalaxyRepositoryOptions(WatchDeployEvents: false),
new GalaxyReconnectOptions(ReplayOnSessionLost: replay));
private static MxEvent DataChange(int handle, double value) => new()
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { DoubleValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
};
private static GalaxyDriver NewDriver(bool replay, HandleSubscriber subscriber) =>
new("g", Opts(replay), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
{
RecreateOverrideForTests = _ => Task.CompletedTask,
};
private static async Task ReconnectAsync(GalaxyDriver driver)
{
await driver.InvokeReopenForTestAsync(TestContext.Current.CancellationToken);
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
}
///
/// The reconnect publishes BadNoCommunication for every tag of every stranded subscription, under
/// that subscription's own handle, and degrades health with a message naming the setting.
///
[Fact]
public async Task Replay_disabled_reconnect_publishes_BadNoCommunication_and_degrades()
{
var subscriber = new HandleSubscriber();
using var driver = NewDriver(replay: false, subscriber);
var events = new ConcurrentQueue();
driver.OnDataChange += (_, e) => events.Enqueue(e);
var ab = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.A", "Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken);
var c = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.C"], TimeSpan.Zero, TestContext.Current.CancellationToken);
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
var callsBefore = Volatile.Read(ref subscriber.SubscribeCalls);
var lastReadBefore = driver.GetHealth().LastSuccessfulRead;
await ReconnectAsync(driver);
Volatile.Read(ref subscriber.SubscribeCalls).ShouldBe(callsBefore, "replay is disabled — nothing is re-advised");
var published = events.ToArray();
published.Select(e => (e.FullReference, ((GalaxySubscriptionHandle)e.SubscriptionHandle).SubscriptionId))
.OrderBy(x => x.FullReference)
.ShouldBe([("Tank.A", ab.SubscriptionId), ("Tank.B", ab.SubscriptionId), ("Tank.C", c.SubscriptionId)]);
published.ShouldAllBe(e => e.Snapshot.StatusCode == BadNoCommunication && e.Snapshot.Value == null);
var health = driver.GetHealth();
health.State.ShouldBe(DriverState.Degraded);
health.LastError.ShouldNotBeNull().ShouldContain("ReplayOnSessionLost");
health.LastError.ShouldContain("2 Galaxy subscription(s)");
health.LastSuccessfulRead.ShouldBe(lastReadBefore, "a synthetic Bad publish is not evidence of a live gateway");
driver.StrandedSubscriptionCountForTests.ShouldBe(2);
}
///
/// Degraded clears exactly when the last stranded subscription is unsubscribed (what the host does on
/// every re-subscribe), and the stranded handles' dead numbers are never sent to the new session.
///
[Fact]
public async Task Degraded_clears_once_every_stranded_subscription_is_unsubscribed()
{
var subscriber = new HandleSubscriber();
using var driver = NewDriver(replay: false, subscriber);
var a = await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
var b = await driver.SubscribeAsync(["Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await ReconnectAsync(driver);
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
await driver.UnsubscribeAsync(a, TestContext.Current.CancellationToken);
driver.GetHealth().State.ShouldBe(DriverState.Degraded, "Tank.B is still stranded");
driver.GetHealth().LastError.ShouldNotBeNull().ShouldContain("1 Galaxy subscription(s)");
await driver.UnsubscribeAsync(b, TestContext.Current.CancellationToken);
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
driver.StrandedSubscriptionCountForTests.ShouldBe(0);
subscriber.Unsubscribed.ShouldBeEmpty("old-session handle numbers must never reach the recreated session");
}
///
/// A subscribe after the reconnect is an ordinary new-session subscription: it receives data, is
/// not counted as stranded, and keeps working once the stranded ones are gone.
///
[Fact]
public async Task A_new_subscribe_after_the_reconnect_works_and_is_not_stranded()
{
var subscriber = new HandleSubscriber();
using var driver = NewDriver(replay: false, subscriber);
var delivered = new ConcurrentQueue();
driver.OnDataChange += (_, e) => delivered.Enqueue(e);
var old = await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await ReconnectAsync(driver);
driver.StrandedSubscriptionCountForTests.ShouldBe(1);
var fresh = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
driver.StrandedSubscriptionCountForTests.ShouldBe(1, "the new subscription holds live new-session handles");
var freshHandle = subscriber.LastHandle;
subscriber.Emit(DataChange(freshHandle, 7.5));
await TestWait.UntilAsync(
() => delivered.Any(e => e.Snapshot.StatusCode == 0 && (double?)e.Snapshot.Value == 7.5),
"a subscribe after the reconnect must receive data");
delivered.Last().SubscriptionHandle.ShouldBe(fresh);
await driver.UnsubscribeAsync(old, TestContext.Current.CancellationToken);
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
subscriber.Unsubscribed.ShouldBeEmpty();
await driver.UnsubscribeAsync(fresh, TestContext.Current.CancellationToken);
subscriber.Unsubscribed.ToArray().ShouldBe(new[] { freshHandle }, "the live subscription unadvises normally");
}
/// Default replay re-advises everything: no Bad publish, no Degraded.
[Fact]
public async Task Replay_enabled_reconnect_publishes_no_Bad_and_stays_undegraded()
{
var subscriber = new HandleSubscriber();
using var driver = NewDriver(replay: true, subscriber);
var events = new ConcurrentQueue();
driver.OnDataChange += (_, e) => events.Enqueue(e);
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await ReconnectAsync(driver);
// Positive control: the replayed handle delivers, so the publish path was live while we looked.
subscriber.Emit(DataChange(subscriber.LastHandle, 1.0));
await TestWait.UntilAsync(() => !events.IsEmpty, "the replayed subscription must deliver");
events.ShouldAllBe(e => e.Snapshot.StatusCode != BadNoCommunication);
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
driver.StrandedSubscriptionCountForTests.ShouldBe(0);
}
/// A second reconnect re-asserts the stranded state (still one entry per subscription).
[Fact]
public async Task A_second_reconnect_restates_the_same_stranded_subscriptions()
{
var subscriber = new HandleSubscriber();
using var driver = NewDriver(replay: false, subscriber);
var events = new ConcurrentQueue();
driver.OnDataChange += (_, e) => events.Enqueue(e);
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await ReconnectAsync(driver);
await ReconnectAsync(driver);
events.Count.ShouldBe(2);
events.ShouldAllBe(e => e.Snapshot.StatusCode == BadNoCommunication);
driver.StrandedSubscriptionCountForTests.ShouldBe(1);
}
/// The runtime teardown forgets the stranded set along with the registry.
[Fact]
public async Task Shutdown_clears_the_stranded_state()
{
var subscriber = new HandleSubscriber();
using var driver = NewDriver(replay: false, subscriber);
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await ReconnectAsync(driver);
driver.StrandedSubscriptionCountForTests.ShouldBe(1);
await driver.ShutdownAsync(TestContext.Current.CancellationToken);
driver.StrandedSubscriptionCountForTests.ShouldBe(0);
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
}
/// Hands out monotonic item handles, records unsubscribes, and streams emitted events.
private sealed class HandleSubscriber : IGalaxySubscriber
{
private readonly Channel _events = Channel.CreateUnbounded();
private int _nextHandle = 100;
/// Number of SubscribeBulk calls.
public int SubscribeCalls;
/// Gets the most recently issued item handle.
public int LastHandle => Volatile.Read(ref _nextHandle);
/// Gets every item handle passed to UnsubscribeBulk.
public ConcurrentQueue Unsubscribed { get; } = new();
/// Pushes an event onto the stream.
/// The event.
public void Emit(MxEvent ev) => _events.Writer.TryWrite(ev).ShouldBeTrue();
///
public Task> SubscribeBulkAsync(
IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
Interlocked.Increment(ref SubscribeCalls);
IReadOnlyList results = [.. fullReferences.Select(r => new SubscribeResult
{
TagAddress = r,
ItemHandle = Interlocked.Increment(ref _nextHandle),
WasSuccessful = true,
})];
return Task.FromResult(results);
}
///
public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken)
{
foreach (var h in itemHandles) Unsubscribed.Enqueue(h);
return Task.CompletedTask;
}
///
public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
}
}