lmxopcua/tests/ZB.MOM.WW.OtOpcUa.Tests.v1Archive/MxAccess/GalaxyRuntimeProbeManagerTests.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Host.Domain;
using ZB.MOM.WW.OtOpcUa.Host.MxAccess;
using ZB.MOM.WW.OtOpcUa.Tests.Helpers;

namespace ZB.MOM.WW.OtOpcUa.Tests.MxAccess
{
    /// <summary>
    ///     Exhaustive coverage of the runtime host probe manager: state machine, sync diff,
    ///     transport gating, unknown-resolution timeout, and transition callbacks.
    /// </summary>
    public class GalaxyRuntimeProbeManagerTests
    {
        // ---------- State transitions ----------

        [Fact]
        public async Task Sync_WithMixedRuntimeHosts_AddsProbesAndEntriesInUnknown()
        {
            var (client, clock) = (new FakeMxAccessClient(), new Clock());
            var (stopSpy, runSpy) = (new List<int>(), new List<int>());
            using var sut = Sut(client, 15, stopSpy, runSpy, clock);

            await sut.SyncAsync(new[]
            {
                Platform(10, "DevPlatform"),
                Engine(20, "DevAppEngine"),
                UserObject(30, "TestMachine_001")
            });

            sut.ActiveProbeCount.ShouldBe(2);
            var snap = sut.GetSnapshot();
            snap.Select(s => s.ObjectName).ShouldBe(new[] { "DevAppEngine", "DevPlatform" });
            snap.All(s => s.State == GalaxyRuntimeState.Unknown).ShouldBeTrue();
            snap.First(s => s.ObjectName == "DevPlatform").Kind.ShouldBe("$WinPlatform");
            snap.First(s => s.ObjectName == "DevAppEngine").Kind.ShouldBe("$AppEngine");
            stopSpy.ShouldBeEmpty();
            runSpy.ShouldBeEmpty();
        }

        [Fact]
        public async Task HandleProbeUpdate_FirstGoodCallback_TransitionsUnknownToRunning()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            var handled = sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            handled.ShouldBeTrue();
            var entry = sut.GetSnapshot().Single();
            entry.State.ShouldBe(GalaxyRuntimeState.Running);
            entry.LastScanState.ShouldBe(true);
            entry.GoodUpdateCount.ShouldBe(1);
            entry.FailureCount.ShouldBe(0);
            entry.LastError.ShouldBeNull();
            // Unknown → Running is startup initialization, not a recovery — the onHostRunning
            // callback is reserved for Stopped → Running transitions so the node manager does
            // not wipe Bad status set by a concurrently-stopping sibling host on the same variable.
            runSpy.ShouldBeEmpty();
            stopSpy.ShouldBeEmpty();
        }

        [Fact]
        public async Task HandleProbeUpdate_ScanStateFalse_TransitionsRunningToStopped()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            stopSpy.Clear(); runSpy.Clear();

            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));

            var entry = sut.GetSnapshot().Single();
            entry.State.ShouldBe(GalaxyRuntimeState.Stopped);
            entry.LastScanState.ShouldBe(false);
            entry.FailureCount.ShouldBe(1);
            entry.LastError!.ShouldContain("OffScan");
            stopSpy.ShouldBe(new[] { 20 });
            runSpy.ShouldBeEmpty();
        }

        [Fact]
        public async Task HandleProbeUpdate_BadQualityCallback_TransitionsRunningToStopped()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Platform(10, "DevPlatform") });
            sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Good(true));
            stopSpy.Clear();

            sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Bad(Quality.BadCommFailure));

            var entry = sut.GetSnapshot().Single();
            entry.State.ShouldBe(GalaxyRuntimeState.Stopped);
            entry.LastError!.ShouldContain("bad quality");
            stopSpy.ShouldBe(new[] { 10 });
        }

        [Fact]
        public async Task HandleProbeUpdate_RecoveryAfterStopped_FiresRunningCallback()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
            runSpy.Clear(); stopSpy.Clear();

            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            runSpy.ShouldBe(new[] { 20 });
            stopSpy.ShouldBeEmpty();
            var entry = sut.GetSnapshot().Single();
            entry.State.ShouldBe(GalaxyRuntimeState.Running);
            entry.LastError.ShouldBeNull();
        }

        [Fact]
        public async Task HandleProbeUpdate_RepeatedRunning_DoesNotRefire()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            // Unknown → Running is silent; subsequent Running updates are idempotent.
            runSpy.ShouldBeEmpty();
            sut.GetSnapshot().Single().GoodUpdateCount.ShouldBe(3);
        }

        [Fact]
        public async Task HandleProbeUpdate_NonProbeAddress_ReturnsFalse()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            var handled = sut.HandleProbeUpdate("UnrelatedObject.Value", Vtq.Good(42));

            handled.ShouldBeFalse();
            sut.GetSnapshot().Single().GoodUpdateCount.ShouldBe(0);
        }

        // ---------- Unknown-resolution timeout ----------

        [Fact]
        public async Task Tick_UnknownBeyondTimeout_TransitionsToStopped()
        {
            var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) };
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy, clock);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            // 16 seconds later — past the 15s timeout
            clock.Now = clock.Now.AddSeconds(16);
            sut.Tick();

            var entry = sut.GetSnapshot().Single();
            entry.State.ShouldBe(GalaxyRuntimeState.Stopped);
            entry.LastError!.ShouldContain("unknown-resolution");
            stopSpy.ShouldBe(new[] { 20 });
        }

        [Fact]
        public async Task Tick_UnknownWithinTimeout_DoesNotTransition()
        {
            var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) };
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy, clock);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            clock.Now = clock.Now.AddSeconds(10);
            sut.Tick();

            sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Unknown);
            stopSpy.ShouldBeEmpty();
        }

        [Fact]
        public async Task Tick_RunningHostWithOldCallback_DoesNotTransition()
        {
            // Critical on-change-semantic test: a stably Running host may go minutes or hours
            // without a callback. Tick must NOT time it out on a starvation basis.
            var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) };
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy, clock);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            clock.Now = clock.Now.AddHours(2); // 2 hours of silence
            sut.Tick();

            sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running);
        }

        // ---------- Transport gating ----------

        [Fact]
        public async Task GetSnapshot_WhenTransportDisconnected_ForcesEveryEntryToUnknown()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[]
            {
                Platform(10, "DevPlatform"),
                Engine(20, "DevAppEngine")
            });
            sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Good(true));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));

            client.State = ConnectionState.Disconnected;

            sut.GetSnapshot().All(s => s.State == GalaxyRuntimeState.Unknown).ShouldBeTrue();

            // Underlying state is preserved — restore transport and snapshot reflects reality again.
            client.State = ConnectionState.Connected;
            var restored = sut.GetSnapshot();
            restored.First(s => s.ObjectName == "DevPlatform").State.ShouldBe(GalaxyRuntimeState.Running);
            restored.First(s => s.ObjectName == "DevAppEngine").State.ShouldBe(GalaxyRuntimeState.Stopped);
        }

        // ---------- Sync diff ----------

        [Fact]
        public async Task Sync_WithHostRemoved_UnadvisesProbeAndDropsEntry()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[]
            {
                Platform(10, "DevPlatform"),
                Engine(20, "DevAppEngine")
            });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            await sut.SyncAsync(new[] { Platform(10, "DevPlatform") });

            sut.ActiveProbeCount.ShouldBe(1);
            sut.GetSnapshot().Single().ObjectName.ShouldBe("DevPlatform");
        }

        [Fact]
        public async Task Sync_WithUnchangedHostSet_PreservesExistingState()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            runSpy.Clear();

            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running);
            runSpy.ShouldBeEmpty(); // no re-fire on no-op resync
        }

        [Fact]
        public async Task Sync_FiltersNonRuntimeCategories()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[]
            {
                Platform(10, "DevPlatform"),
                UserObject(30, "TestMachine_001"),
                AreaObject(40, "DEV"),
                Engine(20, "DevAppEngine"),
                UserObject(31, "TestMachine_002")
            });

            sut.ActiveProbeCount.ShouldBe(2); // only the platform + the engine
        }

        // ---------- Dispose ----------

        [Fact]
        public async Task Dispose_UnadvisesEveryActiveProbe()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[]
            {
                Platform(10, "DevPlatform"),
                Engine(20, "DevAppEngine")
            });

            sut.Dispose();

            sut.ActiveProbeCount.ShouldBe(0);
            // After dispose, a Sync is a no-op.
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.ActiveProbeCount.ShouldBe(0);
        }

        [Fact]
        public void Dispose_OnFreshManager_NoOp()
        {
            var client = new FakeMxAccessClient();
            var sut = Sut(client, 15, new List<int>(), new List<int>());

            Should.NotThrow(() => sut.Dispose());
            Should.NotThrow(() => sut.Dispose());
        }

        [Fact]
        public async Task HandleProbeUpdate_AfterDispose_ReturnsFalse()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.Dispose();

            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)).ShouldBeFalse();
        }

        // ---------- IsHostStopped (Read-path short-circuit support) ----------

        [Fact]
        public async Task IsHostStopped_UnknownHost_ReturnsFalse()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            // Never delivered a callback — state is Unknown. Read-path should NOT short-circuit
            // on Unknown because the host might come online any moment.
            sut.IsHostStopped(20).ShouldBeFalse();
        }

        [Fact]
        public async Task IsHostStopped_RunningHost_ReturnsFalse()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            sut.IsHostStopped(20).ShouldBeFalse();
        }

        [Fact]
        public async Task IsHostStopped_StoppedHost_ReturnsTrue()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));

            sut.IsHostStopped(20).ShouldBeTrue();
        }

        [Fact]
        public async Task IsHostStopped_AfterRecovery_ReturnsFalse()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            sut.IsHostStopped(20).ShouldBeFalse();
        }

        [Fact]
        public async Task IsHostStopped_UnknownGobjectId_ReturnsFalse()
        {
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            // Not a probed host — defensive false rather than throwing.
            sut.IsHostStopped(99999).ShouldBeFalse();
        }

        [Fact]
        public async Task IsHostStopped_TransportDisconnected_UsesUnderlyingState()
        {
            // Critical contract: IsHostStopped is intended for the Read-path short-circuit and
            // uses the underlying state directly, NOT the GetSnapshot transport-gated rewrite.
            // When the transport is disconnected, MxAccess reads will fail via the normal error
            // path; we don't want IsHostStopped to double-flag the Read as stopped if the host
            // itself was actually Running before the transport dropped.
            var (client, stopSpy, runSpy) = NewSpyHarness();
            using var sut = Sut(client, 15, stopSpy, runSpy);
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));

            client.State = ConnectionState.Disconnected;

            // Running state preserved — short-circuit does NOT fire during transport outages.
            sut.IsHostStopped(20).ShouldBeFalse();
        }

        // ---------- Subscribe failure rollback (stability review 2026-04-13 Finding 1) ----------

        [Fact]
        public async Task Sync_SubscribeThrows_DoesNotLeavePhantomEntry()
        {
            var client = new FakeMxAccessClient
            {
                SubscribeException = new InvalidOperationException("advise failed")
            };
            var (stopSpy, runSpy) = (new List<int>(), new List<int>());
            using var sut = Sut(client, 15, stopSpy, runSpy);

            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            // A failed SubscribeAsync must not leave a phantom entry that Tick() can later
            // transition from Unknown to Stopped.
            sut.ActiveProbeCount.ShouldBe(0);
            sut.GetSnapshot().ShouldBeEmpty();
            sut.IsHostStopped(20).ShouldBeFalse();
        }

        [Fact]
        public async Task Sync_SubscribeThrows_TickDoesNotFireStopCallback()
        {
            var client = new FakeMxAccessClient
            {
                SubscribeException = new InvalidOperationException("advise failed")
            };
            var clock = new Clock();
            var (stopSpy, runSpy) = (new List<int>(), new List<int>());
            using var sut = Sut(client, 15, stopSpy, runSpy, clock);

            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            // Advance past the unknown timeout — if the rollback were incomplete, Tick() would
            // transition the phantom entry to Stopped and fan out a false host-down signal.
            clock.Now = clock.Now.AddSeconds(30);
            sut.Tick();

            stopSpy.ShouldBeEmpty();
            runSpy.ShouldBeEmpty();
            sut.ActiveProbeCount.ShouldBe(0);
        }

        [Fact]
        public async Task Sync_SubscribeSucceedsAfterRetry_AppearsInSnapshot()
        {
            // After a failed subscribe rolls back cleanly, a subsequent successful SyncAsync
            // against the same host must behave normally.
            var client = new FakeMxAccessClient
            {
                SubscribeException = new InvalidOperationException("first attempt fails")
            };
            var (stopSpy, runSpy) = (new List<int>(), new List<int>());
            using var sut = Sut(client, 15, stopSpy, runSpy);

            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
            sut.ActiveProbeCount.ShouldBe(0);

            // Clear the fault and resync — the host must now appear with Unknown state.
            client.SubscribeException = null;
            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            sut.ActiveProbeCount.ShouldBe(1);
            sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Unknown);
        }

        // ---------- Callback exception safety ----------

        [Fact]
        public async Task TransitionCallback_ThrowsException_DoesNotCorruptState()
        {
            var client = new FakeMxAccessClient();
            Action<int> badCallback = _ => throw new InvalidOperationException("boom");
            using var sut = new GalaxyRuntimeProbeManager(client, 15, badCallback, badCallback);

            await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });

            Should.NotThrow(() => sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)));
            sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running);
        }

        // ---------- Helpers ----------

        private static GalaxyRuntimeProbeManager Sut(
            FakeMxAccessClient client,
            int timeoutSeconds,
            List<int> stopSpy,
            List<int> runSpy,
            Clock? clock = null)
        {
            clock ??= new Clock();
            return new GalaxyRuntimeProbeManager(
                client, timeoutSeconds,
                stopSpy.Add,
                runSpy.Add,
                () => clock.Now);
        }

        private static (FakeMxAccessClient client, List<int> stopSpy, List<int> runSpy) NewSpyHarness()
        {
            return (new FakeMxAccessClient(), new List<int>(), new List<int>());
        }

        private static GalaxyObjectInfo Platform(int id, string name) => new()
        {
            GobjectId = id,
            TagName = name,
            CategoryId = 1,
            HostedByGobjectId = 0
        };

        private static GalaxyObjectInfo Engine(int id, string name) => new()
        {
            GobjectId = id,
            TagName = name,
            CategoryId = 3,
            HostedByGobjectId = 10
        };

        private static GalaxyObjectInfo UserObject(int id, string name) => new()
        {
            GobjectId = id,
            TagName = name,
            CategoryId = 10,
            HostedByGobjectId = 20
        };

        private static GalaxyObjectInfo AreaObject(int id, string name) => new()
        {
            GobjectId = id,
            TagName = name,
            CategoryId = 13,
            IsArea = true,
            HostedByGobjectId = 20
        };

        private sealed class Clock
        {
            public DateTime Now { get; set; } = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
        }
    }
}