using System.Collections.Concurrent; using System.Runtime.CompilerServices; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using Shouldly; using Xunit; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// #862 — with Reconnect.ReplayOnSessionLost=false a gateway reconnect keeps the driver's /// subscriptions but the recreated session carries none of their advises. That used to be invisible: /// every node kept its last value with Good quality and health went back to Healthy. Now each tag of /// each stranded subscription is published BadNoCommunication and the driver reports Degraded /// until those subscriptions are unsubscribed (or the runtime is torn down). /// public sealed class StrandedSubscriptionTests { private const uint BadNoCommunication = 0x80310000u; private static GalaxyDriverOptions Opts(bool replay) => new( new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), new GalaxyMxAccessOptions("OtOpcUa-A"), new GalaxyRepositoryOptions(WatchDeployEvents: false), new GalaxyReconnectOptions(ReplayOnSessionLost: replay)); private static MxEvent DataChange(int handle, double value) => new() { Family = MxEventFamily.OnDataChange, ItemHandle = handle, Value = new MxValue { DoubleValue = value }, Quality = 192, SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), }; private static GalaxyDriver NewDriver(bool replay, HandleSubscriber subscriber) => new("g", Opts(replay), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber) { RecreateOverrideForTests = _ => Task.CompletedTask, }; private static async Task ReconnectAsync(GalaxyDriver driver) { await driver.InvokeReopenForTestAsync(TestContext.Current.CancellationToken); await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken); } /// /// The reconnect publishes BadNoCommunication for every tag of every stranded subscription, under /// that subscription's own handle, and degrades health with a message naming the setting. /// [Fact] public async Task Replay_disabled_reconnect_publishes_BadNoCommunication_and_degrades() { var subscriber = new HandleSubscriber(); using var driver = NewDriver(replay: false, subscriber); var events = new ConcurrentQueue(); driver.OnDataChange += (_, e) => events.Enqueue(e); var ab = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.A", "Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken); var c = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.C"], TimeSpan.Zero, TestContext.Current.CancellationToken); driver.GetHealth().State.ShouldNotBe(DriverState.Degraded); var callsBefore = Volatile.Read(ref subscriber.SubscribeCalls); var lastReadBefore = driver.GetHealth().LastSuccessfulRead; await ReconnectAsync(driver); Volatile.Read(ref subscriber.SubscribeCalls).ShouldBe(callsBefore, "replay is disabled — nothing is re-advised"); var published = events.ToArray(); published.Select(e => (e.FullReference, ((GalaxySubscriptionHandle)e.SubscriptionHandle).SubscriptionId)) .OrderBy(x => x.FullReference) .ShouldBe([("Tank.A", ab.SubscriptionId), ("Tank.B", ab.SubscriptionId), ("Tank.C", c.SubscriptionId)]); published.ShouldAllBe(e => e.Snapshot.StatusCode == BadNoCommunication && e.Snapshot.Value == null); var health = driver.GetHealth(); health.State.ShouldBe(DriverState.Degraded); health.LastError.ShouldNotBeNull().ShouldContain("ReplayOnSessionLost"); health.LastError.ShouldContain("2 Galaxy subscription(s)"); health.LastSuccessfulRead.ShouldBe(lastReadBefore, "a synthetic Bad publish is not evidence of a live gateway"); driver.StrandedSubscriptionCountForTests.ShouldBe(2); } /// /// Degraded clears exactly when the last stranded subscription is unsubscribed (what the host does on /// every re-subscribe), and the stranded handles' dead numbers are never sent to the new session. /// [Fact] public async Task Degraded_clears_once_every_stranded_subscription_is_unsubscribed() { var subscriber = new HandleSubscriber(); using var driver = NewDriver(replay: false, subscriber); var a = await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken); var b = await driver.SubscribeAsync(["Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken); await ReconnectAsync(driver); driver.GetHealth().State.ShouldBe(DriverState.Degraded); await driver.UnsubscribeAsync(a, TestContext.Current.CancellationToken); driver.GetHealth().State.ShouldBe(DriverState.Degraded, "Tank.B is still stranded"); driver.GetHealth().LastError.ShouldNotBeNull().ShouldContain("1 Galaxy subscription(s)"); await driver.UnsubscribeAsync(b, TestContext.Current.CancellationToken); driver.GetHealth().State.ShouldNotBe(DriverState.Degraded); driver.StrandedSubscriptionCountForTests.ShouldBe(0); subscriber.Unsubscribed.ShouldBeEmpty("old-session handle numbers must never reach the recreated session"); } /// /// A subscribe after the reconnect is an ordinary new-session subscription: it receives data, is /// not counted as stranded, and keeps working once the stranded ones are gone. /// [Fact] public async Task A_new_subscribe_after_the_reconnect_works_and_is_not_stranded() { var subscriber = new HandleSubscriber(); using var driver = NewDriver(replay: false, subscriber); var delivered = new ConcurrentQueue(); driver.OnDataChange += (_, e) => delivered.Enqueue(e); var old = await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken); await ReconnectAsync(driver); driver.StrandedSubscriptionCountForTests.ShouldBe(1); var fresh = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken); driver.StrandedSubscriptionCountForTests.ShouldBe(1, "the new subscription holds live new-session handles"); var freshHandle = subscriber.LastHandle; subscriber.Emit(DataChange(freshHandle, 7.5)); await TestWait.UntilAsync( () => delivered.Any(e => e.Snapshot.StatusCode == 0 && (double?)e.Snapshot.Value == 7.5), "a subscribe after the reconnect must receive data"); delivered.Last().SubscriptionHandle.ShouldBe(fresh); await driver.UnsubscribeAsync(old, TestContext.Current.CancellationToken); driver.GetHealth().State.ShouldNotBe(DriverState.Degraded); subscriber.Unsubscribed.ShouldBeEmpty(); await driver.UnsubscribeAsync(fresh, TestContext.Current.CancellationToken); subscriber.Unsubscribed.ToArray().ShouldBe(new[] { freshHandle }, "the live subscription unadvises normally"); } /// Default replay re-advises everything: no Bad publish, no Degraded. [Fact] public async Task Replay_enabled_reconnect_publishes_no_Bad_and_stays_undegraded() { var subscriber = new HandleSubscriber(); using var driver = NewDriver(replay: true, subscriber); var events = new ConcurrentQueue(); driver.OnDataChange += (_, e) => events.Enqueue(e); await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken); await ReconnectAsync(driver); // Positive control: the replayed handle delivers, so the publish path was live while we looked. subscriber.Emit(DataChange(subscriber.LastHandle, 1.0)); await TestWait.UntilAsync(() => !events.IsEmpty, "the replayed subscription must deliver"); events.ShouldAllBe(e => e.Snapshot.StatusCode != BadNoCommunication); driver.GetHealth().State.ShouldNotBe(DriverState.Degraded); driver.StrandedSubscriptionCountForTests.ShouldBe(0); } /// A second reconnect re-asserts the stranded state (still one entry per subscription). [Fact] public async Task A_second_reconnect_restates_the_same_stranded_subscriptions() { var subscriber = new HandleSubscriber(); using var driver = NewDriver(replay: false, subscriber); var events = new ConcurrentQueue(); driver.OnDataChange += (_, e) => events.Enqueue(e); await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken); await ReconnectAsync(driver); await ReconnectAsync(driver); events.Count.ShouldBe(2); events.ShouldAllBe(e => e.Snapshot.StatusCode == BadNoCommunication); driver.StrandedSubscriptionCountForTests.ShouldBe(1); } /// The runtime teardown forgets the stranded set along with the registry. [Fact] public async Task Shutdown_clears_the_stranded_state() { var subscriber = new HandleSubscriber(); using var driver = NewDriver(replay: false, subscriber); await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken); await ReconnectAsync(driver); driver.StrandedSubscriptionCountForTests.ShouldBe(1); await driver.ShutdownAsync(TestContext.Current.CancellationToken); driver.StrandedSubscriptionCountForTests.ShouldBe(0); driver.GetHealth().State.ShouldNotBe(DriverState.Degraded); } /// Hands out monotonic item handles, records unsubscribes, and streams emitted events. private sealed class HandleSubscriber : IGalaxySubscriber { private readonly Channel _events = Channel.CreateUnbounded(); private int _nextHandle = 100; /// Number of SubscribeBulk calls. public int SubscribeCalls; /// Gets the most recently issued item handle. public int LastHandle => Volatile.Read(ref _nextHandle); /// Gets every item handle passed to UnsubscribeBulk. public ConcurrentQueue Unsubscribed { get; } = new(); /// Pushes an event onto the stream. /// The event. public void Emit(MxEvent ev) => _events.Writer.TryWrite(ev).ShouldBeTrue(); /// public Task> SubscribeBulkAsync( IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) { Interlocked.Increment(ref SubscribeCalls); IReadOnlyList results = [.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = Interlocked.Increment(ref _nextHandle), WasSuccessful = true, })]; return Task.FromResult(results); } /// public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) { foreach (var h in itemHandles) Unsubscribed.Enqueue(h); return Task.CompletedTask; } /// public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } } }