using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Shouldly; using Xunit; using ZB.MOM.WW.LmxOpcUa.Host.Domain; using ZB.MOM.WW.LmxOpcUa.Host.MxAccess; using ZB.MOM.WW.LmxOpcUa.Tests.Helpers; namespace ZB.MOM.WW.LmxOpcUa.Tests.MxAccess { /// /// Exhaustive coverage of the runtime host probe manager: state machine, sync diff, /// transport gating, unknown-resolution timeout, and transition callbacks. /// public class GalaxyRuntimeProbeManagerTests { // ---------- State transitions ---------- [Fact] public async Task Sync_WithMixedRuntimeHosts_AddsProbesAndEntriesInUnknown() { var (client, clock) = (new FakeMxAccessClient(), new Clock()); var (stopSpy, runSpy) = (new List(), new List()); using var sut = Sut(client, 15, stopSpy, runSpy, clock); await sut.SyncAsync(new[] { Platform(10, "DevPlatform"), Engine(20, "DevAppEngine"), UserObject(30, "TestMachine_001") }); sut.ActiveProbeCount.ShouldBe(2); var snap = sut.GetSnapshot(); snap.Select(s => s.ObjectName).ShouldBe(new[] { "DevAppEngine", "DevPlatform" }); snap.All(s => s.State == GalaxyRuntimeState.Unknown).ShouldBeTrue(); snap.First(s => s.ObjectName == "DevPlatform").Kind.ShouldBe("$WinPlatform"); snap.First(s => s.ObjectName == "DevAppEngine").Kind.ShouldBe("$AppEngine"); stopSpy.ShouldBeEmpty(); runSpy.ShouldBeEmpty(); } [Fact] public async Task HandleProbeUpdate_FirstGoodCallback_TransitionsUnknownToRunning() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); var handled = sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); handled.ShouldBeTrue(); var entry = sut.GetSnapshot().Single(); entry.State.ShouldBe(GalaxyRuntimeState.Running); entry.LastScanState.ShouldBe(true); entry.GoodUpdateCount.ShouldBe(1); entry.FailureCount.ShouldBe(0); entry.LastError.ShouldBeNull(); // Unknown → Running is startup initialization, not a recovery — the onHostRunning // callback is reserved for Stopped → Running transitions so the node manager does // not wipe Bad status set by a concurrently-stopping sibling host on the same variable. runSpy.ShouldBeEmpty(); stopSpy.ShouldBeEmpty(); } [Fact] public async Task HandleProbeUpdate_ScanStateFalse_TransitionsRunningToStopped() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); stopSpy.Clear(); runSpy.Clear(); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false)); var entry = sut.GetSnapshot().Single(); entry.State.ShouldBe(GalaxyRuntimeState.Stopped); entry.LastScanState.ShouldBe(false); entry.FailureCount.ShouldBe(1); entry.LastError!.ShouldContain("OffScan"); stopSpy.ShouldBe(new[] { 20 }); runSpy.ShouldBeEmpty(); } [Fact] public async Task HandleProbeUpdate_BadQualityCallback_TransitionsRunningToStopped() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Platform(10, "DevPlatform") }); sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Good(true)); stopSpy.Clear(); sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Bad(Quality.BadCommFailure)); var entry = sut.GetSnapshot().Single(); entry.State.ShouldBe(GalaxyRuntimeState.Stopped); entry.LastError!.ShouldContain("bad quality"); stopSpy.ShouldBe(new[] { 10 }); } [Fact] public async Task HandleProbeUpdate_RecoveryAfterStopped_FiresRunningCallback() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false)); runSpy.Clear(); stopSpy.Clear(); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); runSpy.ShouldBe(new[] { 20 }); stopSpy.ShouldBeEmpty(); var entry = sut.GetSnapshot().Single(); entry.State.ShouldBe(GalaxyRuntimeState.Running); entry.LastError.ShouldBeNull(); } [Fact] public async Task HandleProbeUpdate_RepeatedRunning_DoesNotRefire() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); // Unknown → Running is silent; subsequent Running updates are idempotent. runSpy.ShouldBeEmpty(); sut.GetSnapshot().Single().GoodUpdateCount.ShouldBe(3); } [Fact] public async Task HandleProbeUpdate_NonProbeAddress_ReturnsFalse() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); var handled = sut.HandleProbeUpdate("UnrelatedObject.Value", Vtq.Good(42)); handled.ShouldBeFalse(); sut.GetSnapshot().Single().GoodUpdateCount.ShouldBe(0); } // ---------- Unknown-resolution timeout ---------- [Fact] public async Task Tick_UnknownBeyondTimeout_TransitionsToStopped() { var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) }; var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy, clock); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); // 16 seconds later — past the 15s timeout clock.Now = clock.Now.AddSeconds(16); sut.Tick(); var entry = sut.GetSnapshot().Single(); entry.State.ShouldBe(GalaxyRuntimeState.Stopped); entry.LastError!.ShouldContain("unknown-resolution"); stopSpy.ShouldBe(new[] { 20 }); } [Fact] public async Task Tick_UnknownWithinTimeout_DoesNotTransition() { var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) }; var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy, clock); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); clock.Now = clock.Now.AddSeconds(10); sut.Tick(); sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Unknown); stopSpy.ShouldBeEmpty(); } [Fact] public async Task Tick_RunningHostWithOldCallback_DoesNotTransition() { // Critical on-change-semantic test: a stably Running host may go minutes or hours // without a callback. Tick must NOT time it out on a starvation basis. var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) }; var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy, clock); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); clock.Now = clock.Now.AddHours(2); // 2 hours of silence sut.Tick(); sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running); } // ---------- Transport gating ---------- [Fact] public async Task GetSnapshot_WhenTransportDisconnected_ForcesEveryEntryToUnknown() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Platform(10, "DevPlatform"), Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Good(true)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false)); client.State = ConnectionState.Disconnected; sut.GetSnapshot().All(s => s.State == GalaxyRuntimeState.Unknown).ShouldBeTrue(); // Underlying state is preserved — restore transport and snapshot reflects reality again. client.State = ConnectionState.Connected; var restored = sut.GetSnapshot(); restored.First(s => s.ObjectName == "DevPlatform").State.ShouldBe(GalaxyRuntimeState.Running); restored.First(s => s.ObjectName == "DevAppEngine").State.ShouldBe(GalaxyRuntimeState.Stopped); } // ---------- Sync diff ---------- [Fact] public async Task Sync_WithHostRemoved_UnadvisesProbeAndDropsEntry() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Platform(10, "DevPlatform"), Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); await sut.SyncAsync(new[] { Platform(10, "DevPlatform") }); sut.ActiveProbeCount.ShouldBe(1); sut.GetSnapshot().Single().ObjectName.ShouldBe("DevPlatform"); } [Fact] public async Task Sync_WithUnchangedHostSet_PreservesExistingState() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); runSpy.Clear(); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running); runSpy.ShouldBeEmpty(); // no re-fire on no-op resync } [Fact] public async Task Sync_FiltersNonRuntimeCategories() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Platform(10, "DevPlatform"), UserObject(30, "TestMachine_001"), AreaObject(40, "DEV"), Engine(20, "DevAppEngine"), UserObject(31, "TestMachine_002") }); sut.ActiveProbeCount.ShouldBe(2); // only the platform + the engine } // ---------- Dispose ---------- [Fact] public async Task Dispose_UnadvisesEveryActiveProbe() { var (client, stopSpy, runSpy) = NewSpyHarness(); var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Platform(10, "DevPlatform"), Engine(20, "DevAppEngine") }); sut.Dispose(); sut.ActiveProbeCount.ShouldBe(0); // After dispose, a Sync is a no-op. await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.ActiveProbeCount.ShouldBe(0); } [Fact] public void Dispose_OnFreshManager_NoOp() { var client = new FakeMxAccessClient(); var sut = Sut(client, 15, new List(), new List()); Should.NotThrow(() => sut.Dispose()); Should.NotThrow(() => sut.Dispose()); } [Fact] public async Task HandleProbeUpdate_AfterDispose_ReturnsFalse() { var (client, stopSpy, runSpy) = NewSpyHarness(); var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.Dispose(); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)).ShouldBeFalse(); } // ---------- IsHostStopped (Read-path short-circuit support) ---------- [Fact] public async Task IsHostStopped_UnknownHost_ReturnsFalse() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); // Never delivered a callback — state is Unknown. Read-path should NOT short-circuit // on Unknown because the host might come online any moment. sut.IsHostStopped(20).ShouldBeFalse(); } [Fact] public async Task IsHostStopped_RunningHost_ReturnsFalse() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.IsHostStopped(20).ShouldBeFalse(); } [Fact] public async Task IsHostStopped_StoppedHost_ReturnsTrue() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false)); sut.IsHostStopped(20).ShouldBeTrue(); } [Fact] public async Task IsHostStopped_AfterRecovery_ReturnsFalse() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false)); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); sut.IsHostStopped(20).ShouldBeFalse(); } [Fact] public async Task IsHostStopped_UnknownGobjectId_ReturnsFalse() { var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); // Not a probed host — defensive false rather than throwing. sut.IsHostStopped(99999).ShouldBeFalse(); } [Fact] public async Task IsHostStopped_TransportDisconnected_UsesUnderlyingState() { // Critical contract: IsHostStopped is intended for the Read-path short-circuit and // uses the underlying state directly, NOT the GetSnapshot transport-gated rewrite. // When the transport is disconnected, MxAccess reads will fail via the normal error // path; we don't want IsHostStopped to double-flag the Read as stopped if the host // itself was actually Running before the transport dropped. var (client, stopSpy, runSpy) = NewSpyHarness(); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)); client.State = ConnectionState.Disconnected; // Running state preserved — short-circuit does NOT fire during transport outages. sut.IsHostStopped(20).ShouldBeFalse(); } // ---------- Subscribe failure rollback (stability review 2026-04-13 Finding 1) ---------- [Fact] public async Task Sync_SubscribeThrows_DoesNotLeavePhantomEntry() { var client = new FakeMxAccessClient { SubscribeException = new InvalidOperationException("advise failed") }; var (stopSpy, runSpy) = (new List(), new List()); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); // A failed SubscribeAsync must not leave a phantom entry that Tick() can later // transition from Unknown to Stopped. sut.ActiveProbeCount.ShouldBe(0); sut.GetSnapshot().ShouldBeEmpty(); sut.IsHostStopped(20).ShouldBeFalse(); } [Fact] public async Task Sync_SubscribeThrows_TickDoesNotFireStopCallback() { var client = new FakeMxAccessClient { SubscribeException = new InvalidOperationException("advise failed") }; var clock = new Clock(); var (stopSpy, runSpy) = (new List(), new List()); using var sut = Sut(client, 15, stopSpy, runSpy, clock); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); // Advance past the unknown timeout — if the rollback were incomplete, Tick() would // transition the phantom entry to Stopped and fan out a false host-down signal. clock.Now = clock.Now.AddSeconds(30); sut.Tick(); stopSpy.ShouldBeEmpty(); runSpy.ShouldBeEmpty(); sut.ActiveProbeCount.ShouldBe(0); } [Fact] public async Task Sync_SubscribeSucceedsAfterRetry_AppearsInSnapshot() { // After a failed subscribe rolls back cleanly, a subsequent successful SyncAsync // against the same host must behave normally. var client = new FakeMxAccessClient { SubscribeException = new InvalidOperationException("first attempt fails") }; var (stopSpy, runSpy) = (new List(), new List()); using var sut = Sut(client, 15, stopSpy, runSpy); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.ActiveProbeCount.ShouldBe(0); // Clear the fault and resync — the host must now appear with Unknown state. client.SubscribeException = null; await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); sut.ActiveProbeCount.ShouldBe(1); sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Unknown); } // ---------- Callback exception safety ---------- [Fact] public async Task TransitionCallback_ThrowsException_DoesNotCorruptState() { var client = new FakeMxAccessClient(); Action badCallback = _ => throw new InvalidOperationException("boom"); using var sut = new GalaxyRuntimeProbeManager(client, 15, badCallback, badCallback); await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") }); Should.NotThrow(() => sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true))); sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running); } // ---------- Helpers ---------- private static GalaxyRuntimeProbeManager Sut( FakeMxAccessClient client, int timeoutSeconds, List stopSpy, List runSpy, Clock? clock = null) { clock ??= new Clock(); return new GalaxyRuntimeProbeManager( client, timeoutSeconds, stopSpy.Add, runSpy.Add, () => clock.Now); } private static (FakeMxAccessClient client, List stopSpy, List runSpy) NewSpyHarness() { return (new FakeMxAccessClient(), new List(), new List()); } private static GalaxyObjectInfo Platform(int id, string name) => new() { GobjectId = id, TagName = name, CategoryId = 1, HostedByGobjectId = 0 }; private static GalaxyObjectInfo Engine(int id, string name) => new() { GobjectId = id, TagName = name, CategoryId = 3, HostedByGobjectId = 10 }; private static GalaxyObjectInfo UserObject(int id, string name) => new() { GobjectId = id, TagName = name, CategoryId = 10, HostedByGobjectId = 20 }; private static GalaxyObjectInfo AreaObject(int id, string name) => new() { GobjectId = id, TagName = name, CategoryId = 13, IsArea = true, HostedByGobjectId = 20 }; private sealed class Clock { public DateTime Now { get; set; } = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); } } }