using System.Collections.Immutable;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb;
///
/// Review R05:4 — the scripted-alarm comment trail moved off the condition row into its own
/// append-only alarm_condition_comment table. This proves the move kept the property that
/// mattered: the trail still reaches the redundant pair peer, over a real loopback h2c transport
/// through the real fail-closed interceptor, with both nodes initialised by the production
/// LocalDbSetup.OnReady.
///
///
/// State is written and read through the production
/// on each node, so the ordinal keying, the incremental append and the load-side merge under
/// test are the ones production runs.
///
[Collection("LocalDbPairConvergence")]
public sealed class AlarmConditionCommentConvergenceTests
{
private static readonly DateTime T0 = new(2026, 06, 10, 12, 00, 00, DateTimeKind.Utc);
private static LocalDbAlarmConditionStateStore Store(ILocalDb db) =>
new(db, NullLogger.Instance);
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static async Task CommentCountAsync(ILocalDb db, string alarmId) =>
(await Store(db).LoadAsync(alarmId, Ct))?.Comments.Count ?? -1;
[Fact]
public async Task CommentsWrittenOnA_ArriveOnB_AndBsAppendArrivesBackOnA()
{
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
var storeA = Store(pair.A);
var state = AlarmConditionState.Fresh("eq/alarm-1", T0) with
{
Active = AlarmActiveState.Active,
Acked = AlarmAckedState.Unacknowledged,
};
state = Part9StateMachine.ApplyAddComment(state, "jane", "first look", T0.AddMinutes(1)).State;
state = Part9StateMachine.ApplyAcknowledge(state, "bob", "on it", T0.AddMinutes(2)).State;
await storeA.SaveAsync(state, Ct);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CommentCountAsync(pair.B, "eq/alarm-1") == 2,
"node B to hold node A's two comment-trail entries");
var onB = await Store(pair.B).LoadAsync("eq/alarm-1", Ct);
onB.ShouldNotBeNull();
onB.Comments.ShouldBe(state.Comments);
onB.Acked.ShouldBe(AlarmAckedState.Acknowledged);
// The reverse direction, as an incremental append: B adds one entry to the trail it loaded,
// and only that entry has to travel for A to see the full trail.
var appended = Part9StateMachine.ApplyAddComment(onB, "amy", "from B", T0.AddMinutes(3)).State;
await Store(pair.B).SaveAsync(appended, Ct);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CommentCountAsync(pair.A, "eq/alarm-1") == 3,
"node A to receive the entry node B appended");
(await storeA.LoadAsync("eq/alarm-1", Ct))!.Comments.ShouldBe(appended.Comments);
}
[Fact]
public async Task Failover_AfterAMissedCommand_BothTheMissedEntryAndTheSurvivorsNextEntry_ReachBothPeers()
{
// Review I-1, end to end over the real pair:
// 1. a shared trail on both nodes;
// 2. an AddComment reaches only node A (B's command delivery failed) — A's entry replicates
// into B's table, but B's engine never applied it, so B's IN-MEMORY trail is one short;
// 3. A goes away (link down) and the operator acks on the survivor B — B's in-memory Ack
// collides with A's entry at the same ordinal;
// 4. the pair rejoins.
// Both entries must survive on both peers. Before the fix B's Ack was silently dropped.
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
var shared = AlarmConditionState.Fresh("eq/alarm-fo", T0) with
{
Active = AlarmActiveState.Active,
Acked = AlarmAckedState.Unacknowledged,
};
for (var i = 0; i < 3; i++)
shared = Part9StateMachine.ApplyAddComment(shared, "jane", $"note {i}", T0.AddMinutes(i)).State;
await Store(pair.A).SaveAsync(shared, Ct);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 3,
"node B to hold the shared trail");
var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State;
await Store(pair.A).SaveAsync(onlyOnA, Ct);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 4,
"node A's entry to replicate into node B's table");
await pair.StopPassiveAsync();
// B's engine still holds the 3-entry trail it applied; the operator acks there.
var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State;
await Store(pair.B).SaveAsync(ackOnB, Ct);
await pair.RestartPairAsync();
await LocalDbPairHarness.WaitUntilAsync(
async () => await CommentCountAsync(pair.A, "eq/alarm-fo") == 5
&& await CommentCountAsync(pair.B, "eq/alarm-fo") == 5,
"both nodes to hold the shared trail + A's missed-by-B entry + B's ack");
var onA = (await Store(pair.A).LoadAsync("eq/alarm-fo", Ct))!;
var onB = (await Store(pair.B).LoadAsync("eq/alarm-fo", Ct))!;
onA.Comments.ShouldBe(onB.Comments);
onA.Comments[3].ShouldBe(onlyOnA.Comments[3]);
onA.Comments[4].ShouldBe(ackOnB.Comments[3]);
// Held, not merely reached: nothing late duplicates either entry.
(await LocalDbPairHarness.HeldWithinAsync(
async () => await CommentCountAsync(pair.A, "eq/alarm-fo") != 5
|| await CommentCountAsync(pair.B, "eq/alarm-fo") != 5,
TimeSpan.FromSeconds(2)))
.ShouldBeFalse();
}
[Fact]
public async Task BothNodesRecordingTheSameEntry_ConvergeToOneEntry_NotTwo()
{
// Both pair nodes run the engine and both apply every operator command, each stamping the
// entry with its own clock. Ordinal keying is what collapses the two copies to one entry;
// a content-derived key would leave the trail doubled on both nodes.
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
var fresh = AlarmConditionState.Fresh("eq/alarm-2", T0);
var onA = Part9StateMachine.ApplyAddComment(fresh, "jane", "checked", T0.AddSeconds(1)).State;
var onB = Part9StateMachine.ApplyAddComment(fresh, "jane", "checked", T0.AddSeconds(1).AddMilliseconds(7)).State;
await Store(pair.A).SaveAsync(onA, Ct);
await Store(pair.B).SaveAsync(onB, Ct);
// Positive control: a DISTINCT second alarm written only on B. Without it "one entry on
// both" would also hold for two databases that never replicated at all.
var onlyOnB = Part9StateMachine.ApplyAddComment(
AlarmConditionState.Fresh("eq/only-on-b", T0), "amy", "b only", T0.AddSeconds(2)).State;
await Store(pair.B).SaveAsync(onlyOnB, Ct);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CommentCountAsync(pair.A, "eq/only-on-b") == 1,
"node A to receive node B's distinct alarm");
// LWW picks ONE node's copy and both nodes end up holding that same copy.
await LocalDbPairHarness.WaitUntilAsync(
async () =>
{
var a = await Store(pair.A).LoadAsync("eq/alarm-2", Ct);
var b = await Store(pair.B).LoadAsync("eq/alarm-2", Ct);
return a is not null && b is not null
&& a.Comments.Count == 1 && b.Comments.Count == 1
&& a.Comments[0] == b.Comments[0];
},
"both nodes to agree on a single copy of the shared entry");
// Held, not merely reached: nothing late may double the trail on either node.
(await LocalDbPairHarness.HeldWithinAsync(
async () => await CommentCountAsync(pair.A, "eq/alarm-2") != 1
|| await CommentCountAsync(pair.B, "eq/alarm-2") != 1,
TimeSpan.FromSeconds(2)))
.ShouldBeFalse("the shared entry must stay one entry on both nodes, never two");
}
}