using ZB.MOM.WW.LocalDb; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests; /// /// Phase 1 convergence: operation tracking and site events across a real site pair. /// /// /// This is the test that answers the question Phase 1 exists to answer: does a site node /// pair actually stop losing operation-tracking and site-event state? Everything upstream /// of it — schema helpers, DI wiring, the interceptor — can be individually green while the /// pair still fails to converge. /// /// The fixture lives in , shared with the Phase 2 /// convergence suites. /// /// [Collection("LocalDbSitePairConvergence")] public sealed class LocalDbSitePairConvergenceTests : LocalDbSitePairHarness { // ---- data helpers ----------------------------------------------------------------- private static Task WriteTrackingAsync(ILocalDb db, string id, string status, string target) => db.ExecuteAsync( """ INSERT INTO OperationTracking ( TrackedOperationId, Kind, TargetSummary, Status, RetryCount, CreatedAtUtc, UpdatedAtUtc) VALUES (@id, 'ApiCallCached', @target, @status, 0, @now, @now) ON CONFLICT(TrackedOperationId) DO UPDATE SET Status = excluded.Status, TargetSummary = excluded.TargetSummary, UpdatedAtUtc = excluded.UpdatedAtUtc; """, new { id, status, target, now = DateTime.UtcNow.ToString("o") }); private static Task WriteEventAsync(ILocalDb db, string id, string message) => db.ExecuteAsync( """ INSERT INTO site_events (id, timestamp, event_type, severity, source, message) VALUES (@id, @ts, 'script', 'Info', 'convergence-test', @message); """, new { id, ts = DateTimeOffset.UtcNow.ToString("o"), message }); private static async Task ReadTrackingStatusAsync(ILocalDb db, string id) { var rows = await db.QueryAsync( "SELECT Status FROM OperationTracking WHERE TrackedOperationId = @id", static r => r.GetString(0), new { id }); return rows.Count == 0 ? null : rows[0]; } private static Task> ReadEventIdsAsync(ILocalDb db) => db.QueryAsync("SELECT id FROM site_events ORDER BY id", static r => r.GetString(0)); // ---- scenarios -------------------------------------------------------------------- [Fact] public async Task TrackingRow_WrittenOnA_BecomesReadableOnB() { await WriteTrackingAsync(A, "op-a-1", "Submitted", "ERP.GetOrder"); await WaitUntilAsync( async () => await ReadTrackingStatusAsync(B, "op-a-1") == "Submitted", "the tracking row written on node A to appear on node B"); } [Fact] public async Task TrackingRow_WrittenOnB_BecomesReadableOnA() { // Replication is bidirectional even though only A dials: proving the passive node's // writes flow back is what makes a failover in EITHER direction safe. await WriteTrackingAsync(B, "op-b-1", "Submitted", "ERP.GetOrder"); await WaitUntilAsync( async () => await ReadTrackingStatusAsync(A, "op-b-1") == "Submitted", "the tracking row written on node B to appear on node A"); } [Fact] public async Task SameOperation_UpdatedOnBothNodes_ConvergesToOneWinner() { // Last-writer-wins on the primary key. The specific winner is not asserted — that is // the HLC's business — but the two nodes MUST agree, and must agree on a value one of // them actually wrote rather than a merge of both. await WriteTrackingAsync(A, "op-conflict", "Submitted", "ERP.GetOrder"); await WaitUntilAsync( async () => await ReadTrackingStatusAsync(B, "op-conflict") is not null, "the conflicting row to exist on both nodes before it is updated"); await WriteTrackingAsync(A, "op-conflict", "Delivered", "ERP.GetOrder"); await WriteTrackingAsync(B, "op-conflict", "Parked", "ERP.GetOrder"); await WaitUntilAsync( async () => { var a = await ReadTrackingStatusAsync(A, "op-conflict"); var b = await ReadTrackingStatusAsync(B, "op-conflict"); return a is not null && a == b; }, "both nodes to converge on one status for the contended operation"); var winner = await ReadTrackingStatusAsync(A, "op-conflict"); Assert.Contains(winner, new[] { "Delivered", "Parked" }); } [Fact] public async Task EventsLoggedOnBothNodes_ConvergeToTheUnion_WithNoIdCollisions() { // The reason site_events moved to GUID ids. Under the old autoincrement scheme both // nodes would independently mint id=1, id=2, ... and last-writer-wins on the primary // key would silently DESTROY one node's events instead of merging them. The union // count is the assertion that catches that. var idsFromA = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList(); var idsFromB = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList(); foreach (var id in idsFromA) await WriteEventAsync(A, id, "from A"); foreach (var id in idsFromB) await WriteEventAsync(B, id, "from B"); var expected = idsFromA.Concat(idsFromB).OrderBy(x => x, StringComparer.Ordinal).ToList(); await WaitUntilAsync( async () => (await ReadEventIdsAsync(A)).SequenceEqual(expected) && (await ReadEventIdsAsync(B)).SequenceEqual(expected), "both nodes to hold the union of all 10 events"); } [Fact] public async Task PeerOffline_ThenRejoins_CatchesUpOnEverythingItMissed() { // The failover case that motivates Phase 1: one node is down while the other keeps // working, and nothing written during the outage may be lost. await StopPassiveAsync(); var idsDuringOutage = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList(); foreach (var id in idsDuringOutage) await WriteEventAsync(A, id, "written while B was down"); await WriteTrackingAsync(A, "op-during-outage", "Delivered", "ERP.GetOrder"); // Node B's database survived the host teardown (pre-constructed instance), so this is // a genuine rejoin rather than a fresh node. await RestartPairAsync(); await WaitUntilAsync( async () => { var events = await ReadEventIdsAsync(B); return idsDuringOutage.All(events.Contains) && await ReadTrackingStatusAsync(B, "op-during-outage") == "Delivered"; }, "node B to catch up on everything written while it was offline"); } }