lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmConditionCommentConvergenceTests.cs

using System.Collections.Immutable;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;

namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb;

/// <summary>
///     Review R05:4 — the scripted-alarm comment trail moved off the condition row into its own
///     append-only <c>alarm_condition_comment</c> table. This proves the move kept the property that
///     mattered: the trail still reaches the redundant pair peer, over a real loopback h2c transport
///     through the real fail-closed interceptor, with both nodes initialised by the production
///     <c>LocalDbSetup.OnReady</c>.
/// </summary>
/// <remarks>
///     State is written and read through the production <see cref="LocalDbAlarmConditionStateStore"/>
///     on each node, so the ordinal keying, the incremental append and the load-side merge under
///     test are the ones production runs.
/// </remarks>
[Collection("LocalDbPairConvergence")]
public sealed class AlarmConditionCommentConvergenceTests
{
    private static readonly DateTime T0 = new(2026, 06, 10, 12, 00, 00, DateTimeKind.Utc);

    private static LocalDbAlarmConditionStateStore Store(ILocalDb db) =>
        new(db, NullLogger<LocalDbAlarmConditionStateStore>.Instance);

    private static CancellationToken Ct => TestContext.Current.CancellationToken;

    private static async Task<int> CommentCountAsync(ILocalDb db, string alarmId) =>
        (await Store(db).LoadAsync(alarmId, Ct))?.Comments.Count ?? -1;

    [Fact]
    public async Task CommentsWrittenOnA_ArriveOnB_AndBsAppendArrivesBackOnA()
    {
        await using var pair = new LocalDbPairHarness();
        await pair.StartAsync();

        var storeA = Store(pair.A);
        var state = AlarmConditionState.Fresh("eq/alarm-1", T0) with
        {
            Active = AlarmActiveState.Active,
            Acked = AlarmAckedState.Unacknowledged,
        };
        state = Part9StateMachine.ApplyAddComment(state, "jane", "first look", T0.AddMinutes(1)).State;
        state = Part9StateMachine.ApplyAcknowledge(state, "bob", "on it", T0.AddMinutes(2)).State;
        await storeA.SaveAsync(state, Ct);

        await LocalDbPairHarness.WaitUntilAsync(
            async () => await CommentCountAsync(pair.B, "eq/alarm-1") == 2,
            "node B to hold node A's two comment-trail entries");

        var onB = await Store(pair.B).LoadAsync("eq/alarm-1", Ct);
        onB.ShouldNotBeNull();
        onB.Comments.ShouldBe(state.Comments);
        onB.Acked.ShouldBe(AlarmAckedState.Acknowledged);

        // The reverse direction, as an incremental append: B adds one entry to the trail it loaded,
        // and only that entry has to travel for A to see the full trail.
        var appended = Part9StateMachine.ApplyAddComment(onB, "amy", "from B", T0.AddMinutes(3)).State;
        await Store(pair.B).SaveAsync(appended, Ct);

        await LocalDbPairHarness.WaitUntilAsync(
            async () => await CommentCountAsync(pair.A, "eq/alarm-1") == 3,
            "node A to receive the entry node B appended");

        (await storeA.LoadAsync("eq/alarm-1", Ct))!.Comments.ShouldBe(appended.Comments);
    }

    [Fact]
    public async Task Failover_AfterAMissedCommand_BothTheMissedEntryAndTheSurvivorsNextEntry_ReachBothPeers()
    {
        // Review I-1, end to end over the real pair:
        //   1. a shared trail on both nodes;
        //   2. an AddComment reaches only node A (B's command delivery failed) — A's entry replicates
        //      into B's table, but B's engine never applied it, so B's IN-MEMORY trail is one short;
        //   3. A goes away (link down) and the operator acks on the survivor B — B's in-memory Ack
        //      collides with A's entry at the same ordinal;
        //   4. the pair rejoins.
        // Both entries must survive on both peers. Before the fix B's Ack was silently dropped.
        await using var pair = new LocalDbPairHarness();
        await pair.StartAsync();

        var shared = AlarmConditionState.Fresh("eq/alarm-fo", T0) with
        {
            Active = AlarmActiveState.Active,
            Acked = AlarmAckedState.Unacknowledged,
        };
        for (var i = 0; i < 3; i++)
            shared = Part9StateMachine.ApplyAddComment(shared, "jane", $"note {i}", T0.AddMinutes(i)).State;
        await Store(pair.A).SaveAsync(shared, Ct);
        await LocalDbPairHarness.WaitUntilAsync(
            async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 3,
            "node B to hold the shared trail");

        var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State;
        await Store(pair.A).SaveAsync(onlyOnA, Ct);
        await LocalDbPairHarness.WaitUntilAsync(
            async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 4,
            "node A's entry to replicate into node B's table");

        await pair.StopPassiveAsync();

        // B's engine still holds the 3-entry trail it applied; the operator acks there.
        var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State;
        await Store(pair.B).SaveAsync(ackOnB, Ct);

        await pair.RestartPairAsync();

        await LocalDbPairHarness.WaitUntilAsync(
            async () => await CommentCountAsync(pair.A, "eq/alarm-fo") == 5
                        && await CommentCountAsync(pair.B, "eq/alarm-fo") == 5,
            "both nodes to hold the shared trail + A's missed-by-B entry + B's ack");

        var onA = (await Store(pair.A).LoadAsync("eq/alarm-fo", Ct))!;
        var onB = (await Store(pair.B).LoadAsync("eq/alarm-fo", Ct))!;
        onA.Comments.ShouldBe(onB.Comments);
        onA.Comments[3].ShouldBe(onlyOnA.Comments[3]);
        onA.Comments[4].ShouldBe(ackOnB.Comments[3]);

        // Held, not merely reached: nothing late duplicates either entry.
        (await LocalDbPairHarness.HeldWithinAsync(
                async () => await CommentCountAsync(pair.A, "eq/alarm-fo") != 5
                            || await CommentCountAsync(pair.B, "eq/alarm-fo") != 5,
                TimeSpan.FromSeconds(2)))
            .ShouldBeFalse();
    }

    [Fact]
    public async Task BothNodesRecordingTheSameEntry_ConvergeToOneEntry_NotTwo()
    {
        // Both pair nodes run the engine and both apply every operator command, each stamping the
        // entry with its own clock. Ordinal keying is what collapses the two copies to one entry;
        // a content-derived key would leave the trail doubled on both nodes.
        await using var pair = new LocalDbPairHarness();
        await pair.StartAsync();

        var fresh = AlarmConditionState.Fresh("eq/alarm-2", T0);
        var onA = Part9StateMachine.ApplyAddComment(fresh, "jane", "checked", T0.AddSeconds(1)).State;
        var onB = Part9StateMachine.ApplyAddComment(fresh, "jane", "checked", T0.AddSeconds(1).AddMilliseconds(7)).State;
        await Store(pair.A).SaveAsync(onA, Ct);
        await Store(pair.B).SaveAsync(onB, Ct);

        // Positive control: a DISTINCT second alarm written only on B. Without it "one entry on
        // both" would also hold for two databases that never replicated at all.
        var onlyOnB = Part9StateMachine.ApplyAddComment(
            AlarmConditionState.Fresh("eq/only-on-b", T0), "amy", "b only", T0.AddSeconds(2)).State;
        await Store(pair.B).SaveAsync(onlyOnB, Ct);

        await LocalDbPairHarness.WaitUntilAsync(
            async () => await CommentCountAsync(pair.A, "eq/only-on-b") == 1,
            "node A to receive node B's distinct alarm");

        // LWW picks ONE node's copy and both nodes end up holding that same copy.
        await LocalDbPairHarness.WaitUntilAsync(
            async () =>
            {
                var a = await Store(pair.A).LoadAsync("eq/alarm-2", Ct);
                var b = await Store(pair.B).LoadAsync("eq/alarm-2", Ct);
                return a is not null && b is not null
                       && a.Comments.Count == 1 && b.Comments.Count == 1
                       && a.Comments[0] == b.Comments[0];
            },
            "both nodes to agree on a single copy of the shared entry");

        // Held, not merely reached: nothing late may double the trail on either node.
        (await LocalDbPairHarness.HeldWithinAsync(
                async () => await CommentCountAsync(pair.A, "eq/alarm-2") != 1
                            || await CommentCountAsync(pair.B, "eq/alarm-2") != 1,
                TimeSpan.FromSeconds(2)))
            .ShouldBeFalse("the shared entry must stay one entry on both nodes, never two");
    }
}