using System.Collections.Immutable; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb; /// /// Review R05:4 — the scripted-alarm comment trail moved off the condition row into its own /// append-only alarm_condition_comment table. This proves the move kept the property that /// mattered: the trail still reaches the redundant pair peer, over a real loopback h2c transport /// through the real fail-closed interceptor, with both nodes initialised by the production /// LocalDbSetup.OnReady. /// /// /// State is written and read through the production /// on each node, so the ordinal keying, the incremental append and the load-side merge under /// test are the ones production runs. /// [Collection("LocalDbPairConvergence")] public sealed class AlarmConditionCommentConvergenceTests { private static readonly DateTime T0 = new(2026, 06, 10, 12, 00, 00, DateTimeKind.Utc); private static LocalDbAlarmConditionStateStore Store(ILocalDb db) => new(db, NullLogger.Instance); private static CancellationToken Ct => TestContext.Current.CancellationToken; private static async Task CommentCountAsync(ILocalDb db, string alarmId) => (await Store(db).LoadAsync(alarmId, Ct))?.Comments.Count ?? -1; [Fact] public async Task CommentsWrittenOnA_ArriveOnB_AndBsAppendArrivesBackOnA() { await using var pair = new LocalDbPairHarness(); await pair.StartAsync(); var storeA = Store(pair.A); var state = AlarmConditionState.Fresh("eq/alarm-1", T0) with { Active = AlarmActiveState.Active, Acked = AlarmAckedState.Unacknowledged, }; state = Part9StateMachine.ApplyAddComment(state, "jane", "first look", T0.AddMinutes(1)).State; state = Part9StateMachine.ApplyAcknowledge(state, "bob", "on it", T0.AddMinutes(2)).State; await storeA.SaveAsync(state, Ct); await LocalDbPairHarness.WaitUntilAsync( async () => await CommentCountAsync(pair.B, "eq/alarm-1") == 2, "node B to hold node A's two comment-trail entries"); var onB = await Store(pair.B).LoadAsync("eq/alarm-1", Ct); onB.ShouldNotBeNull(); onB.Comments.ShouldBe(state.Comments); onB.Acked.ShouldBe(AlarmAckedState.Acknowledged); // The reverse direction, as an incremental append: B adds one entry to the trail it loaded, // and only that entry has to travel for A to see the full trail. var appended = Part9StateMachine.ApplyAddComment(onB, "amy", "from B", T0.AddMinutes(3)).State; await Store(pair.B).SaveAsync(appended, Ct); await LocalDbPairHarness.WaitUntilAsync( async () => await CommentCountAsync(pair.A, "eq/alarm-1") == 3, "node A to receive the entry node B appended"); (await storeA.LoadAsync("eq/alarm-1", Ct))!.Comments.ShouldBe(appended.Comments); } [Fact] public async Task Failover_AfterAMissedCommand_BothTheMissedEntryAndTheSurvivorsNextEntry_ReachBothPeers() { // Review I-1, end to end over the real pair: // 1. a shared trail on both nodes; // 2. an AddComment reaches only node A (B's command delivery failed) — A's entry replicates // into B's table, but B's engine never applied it, so B's IN-MEMORY trail is one short; // 3. A goes away (link down) and the operator acks on the survivor B — B's in-memory Ack // collides with A's entry at the same ordinal; // 4. the pair rejoins. // Both entries must survive on both peers. Before the fix B's Ack was silently dropped. await using var pair = new LocalDbPairHarness(); await pair.StartAsync(); var shared = AlarmConditionState.Fresh("eq/alarm-fo", T0) with { Active = AlarmActiveState.Active, Acked = AlarmAckedState.Unacknowledged, }; for (var i = 0; i < 3; i++) shared = Part9StateMachine.ApplyAddComment(shared, "jane", $"note {i}", T0.AddMinutes(i)).State; await Store(pair.A).SaveAsync(shared, Ct); await LocalDbPairHarness.WaitUntilAsync( async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 3, "node B to hold the shared trail"); var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State; await Store(pair.A).SaveAsync(onlyOnA, Ct); await LocalDbPairHarness.WaitUntilAsync( async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 4, "node A's entry to replicate into node B's table"); await pair.StopPassiveAsync(); // B's engine still holds the 3-entry trail it applied; the operator acks there. var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State; await Store(pair.B).SaveAsync(ackOnB, Ct); await pair.RestartPairAsync(); await LocalDbPairHarness.WaitUntilAsync( async () => await CommentCountAsync(pair.A, "eq/alarm-fo") == 5 && await CommentCountAsync(pair.B, "eq/alarm-fo") == 5, "both nodes to hold the shared trail + A's missed-by-B entry + B's ack"); var onA = (await Store(pair.A).LoadAsync("eq/alarm-fo", Ct))!; var onB = (await Store(pair.B).LoadAsync("eq/alarm-fo", Ct))!; onA.Comments.ShouldBe(onB.Comments); onA.Comments[3].ShouldBe(onlyOnA.Comments[3]); onA.Comments[4].ShouldBe(ackOnB.Comments[3]); // Held, not merely reached: nothing late duplicates either entry. (await LocalDbPairHarness.HeldWithinAsync( async () => await CommentCountAsync(pair.A, "eq/alarm-fo") != 5 || await CommentCountAsync(pair.B, "eq/alarm-fo") != 5, TimeSpan.FromSeconds(2))) .ShouldBeFalse(); } [Fact] public async Task BothNodesRecordingTheSameEntry_ConvergeToOneEntry_NotTwo() { // Both pair nodes run the engine and both apply every operator command, each stamping the // entry with its own clock. Ordinal keying is what collapses the two copies to one entry; // a content-derived key would leave the trail doubled on both nodes. await using var pair = new LocalDbPairHarness(); await pair.StartAsync(); var fresh = AlarmConditionState.Fresh("eq/alarm-2", T0); var onA = Part9StateMachine.ApplyAddComment(fresh, "jane", "checked", T0.AddSeconds(1)).State; var onB = Part9StateMachine.ApplyAddComment(fresh, "jane", "checked", T0.AddSeconds(1).AddMilliseconds(7)).State; await Store(pair.A).SaveAsync(onA, Ct); await Store(pair.B).SaveAsync(onB, Ct); // Positive control: a DISTINCT second alarm written only on B. Without it "one entry on // both" would also hold for two databases that never replicated at all. var onlyOnB = Part9StateMachine.ApplyAddComment( AlarmConditionState.Fresh("eq/only-on-b", T0), "amy", "b only", T0.AddSeconds(2)).State; await Store(pair.B).SaveAsync(onlyOnB, Ct); await LocalDbPairHarness.WaitUntilAsync( async () => await CommentCountAsync(pair.A, "eq/only-on-b") == 1, "node A to receive node B's distinct alarm"); // LWW picks ONE node's copy and both nodes end up holding that same copy. await LocalDbPairHarness.WaitUntilAsync( async () => { var a = await Store(pair.A).LoadAsync("eq/alarm-2", Ct); var b = await Store(pair.B).LoadAsync("eq/alarm-2", Ct); return a is not null && b is not null && a.Comments.Count == 1 && b.Comments.Count == 1 && a.Comments[0] == b.Comments[0]; }, "both nodes to agree on a single copy of the shared entry"); // Held, not merely reached: nothing late may double the trail on either node. (await LocalDbPairHarness.HeldWithinAsync( async () => await CommentCountAsync(pair.A, "eq/alarm-2") != 1 || await CommentCountAsync(pair.B, "eq/alarm-2") != 1, TimeSpan.FromSeconds(2))) .ShouldBeFalse("the shared entry must stay one entry on both nodes, never two"); } }