using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Shouldly;
using Xunit;
using ZB.MOM.WW.LmxOpcUa.Host.Domain;
using ZB.MOM.WW.LmxOpcUa.Host.MxAccess;
using ZB.MOM.WW.LmxOpcUa.Tests.Helpers;
namespace ZB.MOM.WW.LmxOpcUa.Tests.MxAccess
{
///
/// Exhaustive coverage of the runtime host probe manager: state machine, sync diff,
/// transport gating, unknown-resolution timeout, and transition callbacks.
///
public class GalaxyRuntimeProbeManagerTests
{
// ---------- State transitions ----------
[Fact]
public async Task Sync_WithMixedRuntimeHosts_AddsProbesAndEntriesInUnknown()
{
var (client, clock) = (new FakeMxAccessClient(), new Clock());
var (stopSpy, runSpy) = (new List(), new List());
using var sut = Sut(client, 15, stopSpy, runSpy, clock);
await sut.SyncAsync(new[]
{
Platform(10, "DevPlatform"),
Engine(20, "DevAppEngine"),
UserObject(30, "TestMachine_001")
});
sut.ActiveProbeCount.ShouldBe(2);
var snap = sut.GetSnapshot();
snap.Select(s => s.ObjectName).ShouldBe(new[] { "DevAppEngine", "DevPlatform" });
snap.All(s => s.State == GalaxyRuntimeState.Unknown).ShouldBeTrue();
snap.First(s => s.ObjectName == "DevPlatform").Kind.ShouldBe("$WinPlatform");
snap.First(s => s.ObjectName == "DevAppEngine").Kind.ShouldBe("$AppEngine");
stopSpy.ShouldBeEmpty();
runSpy.ShouldBeEmpty();
}
[Fact]
public async Task HandleProbeUpdate_FirstGoodCallback_TransitionsUnknownToRunning()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
var handled = sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
handled.ShouldBeTrue();
var entry = sut.GetSnapshot().Single();
entry.State.ShouldBe(GalaxyRuntimeState.Running);
entry.LastScanState.ShouldBe(true);
entry.GoodUpdateCount.ShouldBe(1);
entry.FailureCount.ShouldBe(0);
entry.LastError.ShouldBeNull();
// Unknown → Running is startup initialization, not a recovery — the onHostRunning
// callback is reserved for Stopped → Running transitions so the node manager does
// not wipe Bad status set by a concurrently-stopping sibling host on the same variable.
runSpy.ShouldBeEmpty();
stopSpy.ShouldBeEmpty();
}
[Fact]
public async Task HandleProbeUpdate_ScanStateFalse_TransitionsRunningToStopped()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
stopSpy.Clear(); runSpy.Clear();
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
var entry = sut.GetSnapshot().Single();
entry.State.ShouldBe(GalaxyRuntimeState.Stopped);
entry.LastScanState.ShouldBe(false);
entry.FailureCount.ShouldBe(1);
entry.LastError!.ShouldContain("OffScan");
stopSpy.ShouldBe(new[] { 20 });
runSpy.ShouldBeEmpty();
}
[Fact]
public async Task HandleProbeUpdate_BadQualityCallback_TransitionsRunningToStopped()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Platform(10, "DevPlatform") });
sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Good(true));
stopSpy.Clear();
sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Bad(Quality.BadCommFailure));
var entry = sut.GetSnapshot().Single();
entry.State.ShouldBe(GalaxyRuntimeState.Stopped);
entry.LastError!.ShouldContain("bad quality");
stopSpy.ShouldBe(new[] { 10 });
}
[Fact]
public async Task HandleProbeUpdate_RecoveryAfterStopped_FiresRunningCallback()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
runSpy.Clear(); stopSpy.Clear();
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
runSpy.ShouldBe(new[] { 20 });
stopSpy.ShouldBeEmpty();
var entry = sut.GetSnapshot().Single();
entry.State.ShouldBe(GalaxyRuntimeState.Running);
entry.LastError.ShouldBeNull();
}
[Fact]
public async Task HandleProbeUpdate_RepeatedRunning_DoesNotRefire()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
// Unknown → Running is silent; subsequent Running updates are idempotent.
runSpy.ShouldBeEmpty();
sut.GetSnapshot().Single().GoodUpdateCount.ShouldBe(3);
}
[Fact]
public async Task HandleProbeUpdate_NonProbeAddress_ReturnsFalse()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
var handled = sut.HandleProbeUpdate("UnrelatedObject.Value", Vtq.Good(42));
handled.ShouldBeFalse();
sut.GetSnapshot().Single().GoodUpdateCount.ShouldBe(0);
}
// ---------- Unknown-resolution timeout ----------
[Fact]
public async Task Tick_UnknownBeyondTimeout_TransitionsToStopped()
{
var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) };
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy, clock);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
// 16 seconds later — past the 15s timeout
clock.Now = clock.Now.AddSeconds(16);
sut.Tick();
var entry = sut.GetSnapshot().Single();
entry.State.ShouldBe(GalaxyRuntimeState.Stopped);
entry.LastError!.ShouldContain("unknown-resolution");
stopSpy.ShouldBe(new[] { 20 });
}
[Fact]
public async Task Tick_UnknownWithinTimeout_DoesNotTransition()
{
var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) };
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy, clock);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
clock.Now = clock.Now.AddSeconds(10);
sut.Tick();
sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Unknown);
stopSpy.ShouldBeEmpty();
}
[Fact]
public async Task Tick_RunningHostWithOldCallback_DoesNotTransition()
{
// Critical on-change-semantic test: a stably Running host may go minutes or hours
// without a callback. Tick must NOT time it out on a starvation basis.
var clock = new Clock { Now = new DateTime(2026, 4, 13, 10, 0, 0, DateTimeKind.Utc) };
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy, clock);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
clock.Now = clock.Now.AddHours(2); // 2 hours of silence
sut.Tick();
sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running);
}
// ---------- Transport gating ----------
[Fact]
public async Task GetSnapshot_WhenTransportDisconnected_ForcesEveryEntryToUnknown()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[]
{
Platform(10, "DevPlatform"),
Engine(20, "DevAppEngine")
});
sut.HandleProbeUpdate("DevPlatform.ScanState", Vtq.Good(true));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
client.State = ConnectionState.Disconnected;
sut.GetSnapshot().All(s => s.State == GalaxyRuntimeState.Unknown).ShouldBeTrue();
// Underlying state is preserved — restore transport and snapshot reflects reality again.
client.State = ConnectionState.Connected;
var restored = sut.GetSnapshot();
restored.First(s => s.ObjectName == "DevPlatform").State.ShouldBe(GalaxyRuntimeState.Running);
restored.First(s => s.ObjectName == "DevAppEngine").State.ShouldBe(GalaxyRuntimeState.Stopped);
}
// ---------- Sync diff ----------
[Fact]
public async Task Sync_WithHostRemoved_UnadvisesProbeAndDropsEntry()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[]
{
Platform(10, "DevPlatform"),
Engine(20, "DevAppEngine")
});
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
await sut.SyncAsync(new[] { Platform(10, "DevPlatform") });
sut.ActiveProbeCount.ShouldBe(1);
sut.GetSnapshot().Single().ObjectName.ShouldBe("DevPlatform");
}
[Fact]
public async Task Sync_WithUnchangedHostSet_PreservesExistingState()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
runSpy.Clear();
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running);
runSpy.ShouldBeEmpty(); // no re-fire on no-op resync
}
[Fact]
public async Task Sync_FiltersNonRuntimeCategories()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[]
{
Platform(10, "DevPlatform"),
UserObject(30, "TestMachine_001"),
AreaObject(40, "DEV"),
Engine(20, "DevAppEngine"),
UserObject(31, "TestMachine_002")
});
sut.ActiveProbeCount.ShouldBe(2); // only the platform + the engine
}
// ---------- Dispose ----------
[Fact]
public async Task Dispose_UnadvisesEveryActiveProbe()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[]
{
Platform(10, "DevPlatform"),
Engine(20, "DevAppEngine")
});
sut.Dispose();
sut.ActiveProbeCount.ShouldBe(0);
// After dispose, a Sync is a no-op.
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.ActiveProbeCount.ShouldBe(0);
}
[Fact]
public void Dispose_OnFreshManager_NoOp()
{
var client = new FakeMxAccessClient();
var sut = Sut(client, 15, new List(), new List());
Should.NotThrow(() => sut.Dispose());
Should.NotThrow(() => sut.Dispose());
}
[Fact]
public async Task HandleProbeUpdate_AfterDispose_ReturnsFalse()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.Dispose();
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)).ShouldBeFalse();
}
// ---------- IsHostStopped (Read-path short-circuit support) ----------
[Fact]
public async Task IsHostStopped_UnknownHost_ReturnsFalse()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
// Never delivered a callback — state is Unknown. Read-path should NOT short-circuit
// on Unknown because the host might come online any moment.
sut.IsHostStopped(20).ShouldBeFalse();
}
[Fact]
public async Task IsHostStopped_RunningHost_ReturnsFalse()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.IsHostStopped(20).ShouldBeFalse();
}
[Fact]
public async Task IsHostStopped_StoppedHost_ReturnsTrue()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
sut.IsHostStopped(20).ShouldBeTrue();
}
[Fact]
public async Task IsHostStopped_AfterRecovery_ReturnsFalse()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(false));
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
sut.IsHostStopped(20).ShouldBeFalse();
}
[Fact]
public async Task IsHostStopped_UnknownGobjectId_ReturnsFalse()
{
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
// Not a probed host — defensive false rather than throwing.
sut.IsHostStopped(99999).ShouldBeFalse();
}
[Fact]
public async Task IsHostStopped_TransportDisconnected_UsesUnderlyingState()
{
// Critical contract: IsHostStopped is intended for the Read-path short-circuit and
// uses the underlying state directly, NOT the GetSnapshot transport-gated rewrite.
// When the transport is disconnected, MxAccess reads will fail via the normal error
// path; we don't want IsHostStopped to double-flag the Read as stopped if the host
// itself was actually Running before the transport dropped.
var (client, stopSpy, runSpy) = NewSpyHarness();
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true));
client.State = ConnectionState.Disconnected;
// Running state preserved — short-circuit does NOT fire during transport outages.
sut.IsHostStopped(20).ShouldBeFalse();
}
// ---------- Subscribe failure rollback (stability review 2026-04-13 Finding 1) ----------
[Fact]
public async Task Sync_SubscribeThrows_DoesNotLeavePhantomEntry()
{
var client = new FakeMxAccessClient
{
SubscribeException = new InvalidOperationException("advise failed")
};
var (stopSpy, runSpy) = (new List(), new List());
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
// A failed SubscribeAsync must not leave a phantom entry that Tick() can later
// transition from Unknown to Stopped.
sut.ActiveProbeCount.ShouldBe(0);
sut.GetSnapshot().ShouldBeEmpty();
sut.IsHostStopped(20).ShouldBeFalse();
}
[Fact]
public async Task Sync_SubscribeThrows_TickDoesNotFireStopCallback()
{
var client = new FakeMxAccessClient
{
SubscribeException = new InvalidOperationException("advise failed")
};
var clock = new Clock();
var (stopSpy, runSpy) = (new List(), new List());
using var sut = Sut(client, 15, stopSpy, runSpy, clock);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
// Advance past the unknown timeout — if the rollback were incomplete, Tick() would
// transition the phantom entry to Stopped and fan out a false host-down signal.
clock.Now = clock.Now.AddSeconds(30);
sut.Tick();
stopSpy.ShouldBeEmpty();
runSpy.ShouldBeEmpty();
sut.ActiveProbeCount.ShouldBe(0);
}
[Fact]
public async Task Sync_SubscribeSucceedsAfterRetry_AppearsInSnapshot()
{
// After a failed subscribe rolls back cleanly, a subsequent successful SyncAsync
// against the same host must behave normally.
var client = new FakeMxAccessClient
{
SubscribeException = new InvalidOperationException("first attempt fails")
};
var (stopSpy, runSpy) = (new List(), new List());
using var sut = Sut(client, 15, stopSpy, runSpy);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.ActiveProbeCount.ShouldBe(0);
// Clear the fault and resync — the host must now appear with Unknown state.
client.SubscribeException = null;
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
sut.ActiveProbeCount.ShouldBe(1);
sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Unknown);
}
// ---------- Callback exception safety ----------
[Fact]
public async Task TransitionCallback_ThrowsException_DoesNotCorruptState()
{
var client = new FakeMxAccessClient();
Action badCallback = _ => throw new InvalidOperationException("boom");
using var sut = new GalaxyRuntimeProbeManager(client, 15, badCallback, badCallback);
await sut.SyncAsync(new[] { Engine(20, "DevAppEngine") });
Should.NotThrow(() => sut.HandleProbeUpdate("DevAppEngine.ScanState", Vtq.Good(true)));
sut.GetSnapshot().Single().State.ShouldBe(GalaxyRuntimeState.Running);
}
// ---------- Helpers ----------
private static GalaxyRuntimeProbeManager Sut(
FakeMxAccessClient client,
int timeoutSeconds,
List stopSpy,
List runSpy,
Clock? clock = null)
{
clock ??= new Clock();
return new GalaxyRuntimeProbeManager(
client, timeoutSeconds,
stopSpy.Add,
runSpy.Add,
() => clock.Now);
}
private static (FakeMxAccessClient client, List stopSpy, List runSpy) NewSpyHarness()
{
return (new FakeMxAccessClient(), new List(), new List());
}
private static GalaxyObjectInfo Platform(int id, string name) => new()
{
GobjectId = id,
TagName = name,
CategoryId = 1,
HostedByGobjectId = 0
};
private static GalaxyObjectInfo Engine(int id, string name) => new()
{
GobjectId = id,
TagName = name,
CategoryId = 3,
HostedByGobjectId = 10
};
private static GalaxyObjectInfo UserObject(int id, string name) => new()
{
GobjectId = id,
TagName = name,
CategoryId = 10,
HostedByGobjectId = 20
};
private static GalaxyObjectInfo AreaObject(int id, string name) => new()
{
GobjectId = id,
TagName = name,
CategoryId = 13,
IsArea = true,
HostedByGobjectId = 20
};
private sealed class Clock
{
public DateTime Now { get; set; } = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
}
}
}