lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs

using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;

namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;

/// <summary>
///     #865: the platform ScanState probes' initial value. The gateway pushes an advise's current value
///     once, as its first event, and a FRESH <c>StreamEvents</c> (<c>after_worker_sequence = 0</c>) never
///     replays — so a probe advised before the driver's event stream is attached never gets a value, and
///     ScanState rarely changes again. The event pump used to start only on the first Read/Subscribe,
///     so a driver with no authored tags (or any driver before its first read) had no host statuses.
/// </summary>
public sealed class ProbeInitialValueTests
{
    private static readonly string[] Platforms = ["CTRX_DEV", "DevWinPlatform", "DevAppEngine"];

    private static GalaxyDriverOptions Opts() => new(
        new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
        new GalaxyMxAccessOptions("ProbeInitialValue"),
        new GalaxyRepositoryOptions(WatchDeployEvents: false),
        new GalaxyReconnectOptions());

    private sealed class FixedPlatformSource(params string[] platforms) : IGalaxyPlatformSource
    {
        public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
            => Task.FromResult<IReadOnlyList<string>>(platforms);
    }

    /// <summary>
    ///     Models the gateway's event semantics: every advise pushes its current value exactly once, and
    ///     only to a stream that is attached at that moment (no replay for a fresh stream). Records the
    ///     order in which the stream was requested and the advises were made.
    /// </summary>
    private sealed class GatewayLikeSubscriber : IGalaxySubscriber
    {
        private readonly Lock _gate = new();
        private Channel<MxEvent>? _attached;
        private int _nextHandle;

        /// <summary>"stream" per StreamEvents call, "subscribe:&lt;refs&gt;" per SubscribeBulk, in call order.</summary>
        public List<string> Calls { get; } = [];

        /// <summary>Initial values pushed while no stream was attached — lost, as on the real gateway.</summary>
        public int DroppedInitialValues;

        private int _streams;

        /// <summary>Streams beyond this count end immediately after attaching (a gateway refusing/ending them).</summary>
        public int StreamsThatStayOpen = int.MaxValue;

        /// <summary>The ScanState value the next advise reports.</summary>
        public volatile bool ScanStateRunning = true;

        public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
            IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
        {
            var results = new List<SubscribeResult>();
            lock (_gate)
            {
                Calls.Add("subscribe:" + string.Join(",", fullReferences));
                foreach (var r in fullReferences)
                {
                    var h = ++_nextHandle;
                    results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
                    var initial = new MxEvent
                    {
                        Family = MxEventFamily.OnDataChange,
                        ItemHandle = h,
                        Value = new MxValue { BoolValue = ScanStateRunning },
                        Quality = 192,
                        SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
                    };
                    if (_attached is null || !_attached.Writer.TryWrite(initial)) Interlocked.Increment(ref DroppedInitialValues);
                }
            }
            return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
        }

        public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;

        public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
        {
            // Attached synchronously on the first MoveNext — the point a real gRPC call is issued.
            var mine = Channel.CreateUnbounded<MxEvent>();
            lock (_gate)
            {
                Calls.Add("stream");
                _attached = mine;
            }
            if (Interlocked.Increment(ref _streams) > StreamsThatStayOpen)
            {
                lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; }
                yield break; // the gateway completed this stream at once
            }
            try
            {
                await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
                {
                    yield return ev;
                }
            }
            finally
            {
                lock (_gate)
                {
                    if (ReferenceEquals(_attached, mine)) _attached = null;
                }
            }
        }

        /// <summary>Fault the currently attached stream (the gateway dropping it).</summary>
        public void FaultAttachedStream(Exception cause)
        {
            lock (_gate) _attached.ShouldNotBeNull("a stream must be attached").Writer.TryComplete(cause);
        }

        public string[] CallsSnapshot()
        {
            lock (_gate) return [.. Calls];
        }
    }

    private static HostState? StateOf(GalaxyDriver driver, string host) =>
        driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;

    private static string Describe(GalaxyDriver driver, GatewayLikeSubscriber sub) =>
        $"hosts=[{string.Join(", ", driver.GetHostStatuses().Select(s => $"{s.HostName}={s.State}"))}] " +
        $"calls=[{string.Join(" | ", sub.CallsSnapshot())}] dropped={sub.DroppedInitialValues} pump={driver.HasEventPumpForTests}";

    /// <summary>
    ///     A driver with NO authored tags — nothing ever Reads or Subscribes — still reports every seeded
    ///     platform with its initial ScanState. Before #865 the pump never started, every initial value
    ///     was pushed to no stream, and only the transport entry ever appeared.
    /// </summary>
    [Fact]
    public async Task Driver_with_no_authored_tags_reports_every_platform_from_its_initial_ScanState()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
            platformSource: new FixedPlatformSource(Platforms));

        await driver.InitializeAsync("{}", CancellationToken.None);

        await TestWait.UntilAsync(
            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
            "every seeded platform must surface Running from its initial ScanState, with no Read/Subscribe",
            () => Describe(driver, sub));
        sub.DroppedInitialValues.ShouldBe(0, "no probe's initial value may be pushed before the stream is attached");
        driver.TrackedSubscriptionCountForTests.ShouldBe(1, "only the probe entry — no data subscription was ever made");
    }

    /// <summary>
    ///     Ordering: the event stream is requested BEFORE the first probe SubscribeBulk. On the old code
    ///     the probe advise went out with no stream at all.
    /// </summary>
    [Fact]
    public async Task Probe_subscribe_is_issued_only_after_the_event_stream_is_requested()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
            platformSource: new FixedPlatformSource(Platforms));

        await driver.InitializeAsync("{}", CancellationToken.None);
        await TestWait.UntilAsync(
            () => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == Platforms.Length,
            "positive control: the background seed must advise every platform",
            () => Describe(driver, sub));

        var calls = sub.CallsSnapshot();
        var firstSubscribe = Array.FindIndex(calls, c => c.StartsWith("subscribe:", StringComparison.Ordinal));
        var firstStream = Array.IndexOf(calls, "stream");
        firstSubscribe.ShouldBeGreaterThanOrEqualTo(0);
        firstStream.ShouldBeGreaterThanOrEqualTo(0, $"the probe seed must start the event stream; calls=[{string.Join(" | ", calls)}]");
        firstStream.ShouldBeLessThan(firstSubscribe, $"the stream must be requested before the probe advise; calls=[{string.Join(" | ", calls)}]");
    }

    /// <summary>
    ///     A reconnect replay re-advises the probes on the recreated session; their new initial values must
    ///     arrive too (the replay restarts the pump first). The value flips to Stopped so the presence wait
    ///     can only pass on a post-replay event.
    /// </summary>
    [Fact]
    public async Task Reconnect_replay_delivers_the_re_advised_probes_initial_values()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
            platformSource: new FixedPlatformSource(Platforms));
        await driver.InitializeAsync("{}", CancellationToken.None);
        await TestWait.UntilAsync(
            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
            "positive control: the initial seed reports Running",
            () => Describe(driver, sub));

        sub.ScanStateRunning = false;
        await driver.InvokeReplayForTestAsync(CancellationToken.None);

        await TestWait.UntilAsync(
            () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
            "the re-advised probes' initial values (Stopped) must reach the host statuses after a replay",
            () => Describe(driver, sub));
        sub.DroppedInitialValues.ShouldBe(0);
    }

    /// <summary>
    ///     Review finding: a probe advised in the reopen → replay window (the session was swapped, the old
    ///     pump still streams the dead session) must still get its initial value. The advise waits for the
    ///     replay's restarted pump instead of advising onto the stale one (its value would route under the
    ///     old session and be lost, and the replay skips a current-session handle) or starting a competing
    ///     stream (the gateway's default single-subscriber mode refuses a second attach).
    /// </summary>
    [Fact]
    public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
            platformSource: new FixedPlatformSource(Platforms))
        {
            RecreateOverrideForTests = _ => Task.CompletedTask,
            // Generous: the window advise must still be waiting when the replay below starts the pump, even
            // on a stalled agent; the wait ends the moment that pump exists.
            StreamRequestWait = TimeSpan.FromSeconds(20),
        };
        await driver.InitializeAsync("{}", CancellationToken.None);
        await TestWait.UntilAsync(
            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
            "positive control: the initial seed reports Running",
            () => Describe(driver, sub));
        var oldPump = driver.EventPumpInstanceForTests;

        await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet
        sub.ScanStateRunning = false;
        var callsBeforeWindow = sub.CallsSnapshot().Length;
        // The window advise must WAIT for the replay's current-session pump rather than advise onto the
        // stale one (or start a competing stream).
        var sync = driver.ProbeWatcherForTests.ShouldNotBeNull()
            .SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None);

        await driver.InvokeReplayForTestAsync(CancellationToken.None);
        await TestWait.ForAsync(sync, "the window advise completes once the replay restarted the pump");

        await TestWait.UntilAsync(
            () => StateOf(driver, "LatePlatform") == HostState.Stopped
                  && Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
            "the window advise and the replay's re-advises all get their initial values",
            () => Describe(driver, sub));
        driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the replay restarted the stale pump");
        var windowCalls = sub.CallsSnapshot()[callsBeforeWindow..];
        var lateAdvise = Array.FindIndex(windowCalls, c => c.Contains("LatePlatform", StringComparison.Ordinal));
        var newStream = Array.IndexOf(windowCalls, "stream");
        newStream.ShouldBeGreaterThanOrEqualTo(0, $"the replay must start a new stream; calls=[{string.Join(" | ", windowCalls)}]");
        lateAdvise.ShouldBeGreaterThan(newStream, $"the window advise must follow the new stream; calls=[{string.Join(" | ", windowCalls)}]");
        windowCalls.Count(c => c == "stream").ShouldBe(1, "no competing stream may be started for the window advise");
        sub.DroppedInitialValues.ShouldBe(0);
    }

    /// <summary>
    ///     The data paths too: a first SubscribeAsync and a first ReadAsync on a driver whose pump was never
    ///     started request the stream BEFORE advising, so their initial values arrive (a read of a value that
    ///     never changes again would otherwise time out).
    /// </summary>
    [Fact]
    public async Task First_subscribe_and_read_advise_only_after_the_stream_is_requested()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
        {
            ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
        };
        await driver.InitializeAsync("{}", CancellationToken.None);
        var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
        driver.OnDataChange += (_, e) => delivered.TrySetResult(e);

        await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
        var calls = sub.CallsSnapshot();
        Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the advise; calls=[{string.Join(" | ", calls)}]");
        (await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub)))
            .FullReference.ShouldBe("Tank01.Level");

        sub.DroppedInitialValues.ShouldBe(0);
    }

    /// <summary>A first ReadAsync on a fresh driver (no pump yet) requests the stream before its advise.</summary>
    [Fact]
    public async Task First_read_advises_only_after_the_stream_is_requested()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
        {
            ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
        };
        await driver.InitializeAsync("{}", CancellationToken.None);
        driver.HasEventPumpForTests.ShouldBeFalse("precondition: the read is the first advise");

        var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None);

        var calls = sub.CallsSnapshot();
        Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the read's advise; calls=[{string.Join(" | ", calls)}]");
        read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)");
        sub.DroppedInitialValues.ShouldBe(0);
    }

    /// <summary>
    ///     The replay's re-advise of DATA subscriptions (which has no per-advise hook) waits for the restarted
    ///     pump's stream, so each re-advised tag's new initial value arrives. The value flips so only a
    ///     post-replay event can satisfy the wait.
    /// </summary>
    [Fact]
    public async Task Reconnect_replay_delivers_re_advised_data_subscriptions_initial_values()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
        await driver.InitializeAsync("{}", CancellationToken.None);
        var last = new System.Collections.Concurrent.ConcurrentDictionary<string, object?>();
        driver.OnDataChange += (_, e) => last[e.FullReference] = e.Snapshot.Value;

        await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
        await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is true,
            "positive control: the subscribe's initial value arrives", () => Describe(driver, sub));

        sub.ScanStateRunning = false; // the fake's next initial value
        await driver.InvokeReplayForTestAsync(CancellationToken.None);

        await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is false,
            "the replay's re-advise must deliver its new initial value",
            () => Describe(driver, sub) + $" last=[{string.Join(",", last.Select(kv => $"{kv.Key}={kv.Value}"))}] parked={driver.ParkedEarlyEventCountForTests} h2=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(2).Select(b => b.FullReference))}] h1=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(1).Select(b => b.FullReference))}]");
        sub.DroppedInitialValues.ShouldBe(0);
    }

    /// <summary>
    ///     A replay whose restarted pump's stream ends at once must FAIL (the supervisor then backs off and
    ///     stays Degraded) rather than succeed onto a dead stream — succeeding went Healthy, and the next
    ///     advise's re-report started a new cycle with no backoff: a tight reconnect loop. Positive control:
    ///     the first replay, whose stream stays open, succeeds.
    /// </summary>
    [Fact]
    public async Task Replay_fails_when_the_restarted_stream_ends_at_once()
    {
        var sub = new GatewayLikeSubscriber { StreamsThatStayOpen = 2 };
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
        await driver.InitializeAsync("{}", CancellationToken.None);
        await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // stream 1

        await driver.InvokeReplayForTestAsync(CancellationToken.None); // stream 2 stays open — succeeds

        await Should.ThrowAsync<InvalidOperationException>(
            () => driver.InvokeReplayForTestAsync(CancellationToken.None), // stream 3 ends at once
            "a replay must not report success over a dead event stream");
    }

    /// <summary>
    ///     #865 review: a stream fault is escalated only from the LIVE pump. A fault from a pump superseded
    ///     by a reopen (its session is no longer current) is ignored — it would fail a good replay — while
    ///     the live pump's faults (before the reopen, and the replay's new pump after it) still escalate.
    /// </summary>
    [Fact]
    public async Task Only_the_live_pumps_stream_faults_are_escalated()
    {
        var sub = new GatewayLikeSubscriber();
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
        {
            RecreateOverrideForTests = _ => Task.CompletedTask,
        };
        await driver.InitializeAsync("{}", CancellationToken.None);
        await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // pump 1

        // Positive control: the live pump's fault escalates.
        sub.FaultAttachedStream(new IOException("drop 1"));
        await TestWait.UntilAsync(() => driver.EscalatedStreamFaultsForTests == 1, "the live pump's fault must escalate");

        // Recover onto pump 2 (the replay restarts the faulted pump), then reopen: pump 2 is now superseded.
        await driver.InvokeReplayForTestAsync(CancellationToken.None);
        var pump2 = driver.EventPumpInstanceForTests.ShouldNotBeNull();
        await driver.InvokeReopenForTestAsync(CancellationToken.None);
        sub.FaultAttachedStream(new IOException("late fault from the superseded pump"));
        await TestWait.UntilAsync(() => ((EventPump)pump2).HasEnded, "the superseded pump's stream must have faulted");

        // Positive control after the absence: the replay's new pump faults and that DOES escalate — so the
        // count can only be 2 if the superseded pump's fault was ignored.
        await driver.InvokeReplayForTestAsync(CancellationToken.None);
        var pump3 = (EventPump)driver.EventPumpInstanceForTests.ShouldNotBeNull();
        pump3.ShouldNotBeSameAs(pump2);
        sub.FaultAttachedStream(new IOException("drop 3"));
        // HasEnded is set only after the fault callback ran, so once both pumps have ended every escalation
        // that will ever happen has happened.
        await TestWait.UntilAsync(() => pump3.HasEnded, "the new live pump's stream must have faulted");
        driver.EscalatedStreamFaultsForTests.ShouldBe(2, "the new live pump's fault escalates; the superseded pump's did not");
    }

    /// <summary>
    ///     A probe sync on a driver whose pump was never started (no platform source, e.g. the
    ///     DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just
    ///     the init path.
    /// </summary>
    [Fact]
    public async Task Probe_sync_without_a_platform_source_still_starts_the_stream_first()
    {
        var sub = new GatewayLikeSubscriber();
        // No platform source: init starts no pump; the probe set arrives through SyncPlatformsAsync.
        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
        await driver.InitializeAsync("{}", CancellationToken.None);
        driver.HasEventPumpForTests.ShouldBeFalse("precondition: nothing has started the pump yet");

        await driver.ProbeWatcherForTests.ShouldNotBeNull().SyncPlatformsAsync(Platforms, CancellationToken.None);

        await TestWait.UntilAsync(
            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
            "the watcher must start the stream before advising, so the initial values arrive",
            () => Describe(driver, sub));
        sub.DroppedInitialValues.ShouldBe(0);
    }
}