ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/SfBufferResyncPredicateTests.cs

using Akka.Actor;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;

/// <summary>
/// N1 regression (review 02 round 2, Critical): the resync authority must use the same
/// oldest-Up predicate as the S&F delivery gate. Divergence scenario = the delivering node
/// is OLDEST but not LEADER (leader = lowest address), the exact state a rolling restart of
/// the lower-address node produces. Pre-fix the delivering node requests a resync from the
/// stale peer and ReplaceAllAsync wipes its live buffer.
/// </summary>
public class SfBufferResyncPredicateTests
{
    [Fact]
    public async Task OldestButNotLeaderNode_KeepsItsBuffer_AndSeedsTheJoiner()
    {
        // Two explicit ports, deliberately assigned so the FIRST-started (oldest,
        // delivering) node has the HIGHER address → the second node is cluster leader.
        var p1 = TwoNodeClusterFixture.GetFreeTcpPort();
        var p2 = TwoNodeClusterFixture.GetFreeTcpPort();
        var (portHigh, portLow) = p1 > p2 ? (p1, p2) : (p2, p1);

        var fixture = await TwoNodeClusterFixture.StartAsync(
            role: "site-int", portA: portHigh, portB: portLow);

        // The S&F stores AND SiteStorageService take an ILocalDb (LocalDb has no in-memory
        // mode), so each node gets its own temp-file local databases. They are disposed
        // AFTER the cluster is shut down — the actors hold connections while the systems
        // are alive — and only then are the files (plus their WAL sidecars) deleted.
        var localDbs = new List<TestLocalDb>();

        try
        {
            // Real S&F storage + replication actor per node, production default predicate
            // (no isActiveOverride) — the exact wiring under test.
            var (storageOldest, _, sfDbOldest, siteDbOldest) =
                await CreateReplicationActorAsync(fixture.NodeA, "oldest");
            localDbs.Add(sfDbOldest);
            localDbs.Add(siteDbOldest);
            var (storageJoiner, _, sfDbJoiner, siteDbJoiner) =
                await CreateReplicationActorAsync(fixture.NodeB, "joiner");
            localDbs.Add(sfDbJoiner);
            localDbs.Add(siteDbJoiner);

            // The delivering (oldest) node has a live buffered row the standby never saw.
            await storageOldest.EnqueueAsync(NewMessage("live-row"));

            // Trigger peer (re)tracking on both sides: each actor got InitialStateAsSnapshot
            // in PreStart, but the enqueue raced it — re-deliver via a fresh MemberUp is not
            // needed; OnPeerTracked already fired on join. The resync exchange is async:
            // wait until the JOINER holds the row (proves the snapshot flowed oldest→joiner,
            // the correct direction). Pre-fix this times out (the joiner, as leader, never
            // requests) AND the oldest node's row is deleted by the stale wipe.
            await AwaitAsync(async () => await storageJoiner.GetMessageByIdAsync("live-row") != null,
                TimeSpan.FromSeconds(20),
                "joiner never received the resync snapshot (resync ran in the wrong direction)");

            // And the delivering node's buffer is untouched — the N1 wipe assertion.
            Assert.NotNull(await storageOldest.GetMessageByIdAsync("live-row"));
        }
        finally
        {
            await fixture.DisposeAsync();

            foreach (var localDb in localDbs)
            {
                var path = localDb.Path;
                localDb.Dispose();
                TestLocalDb.DeleteFiles(path);
            }
        }
    }

    private static async Task<(
        StoreAndForwardStorage Storage, IActorRef Actor, TestLocalDb SfLocalDb, TestLocalDb SiteLocalDb)>
        CreateReplicationActorAsync(ActorSystem node, string tag)
    {
        var sfLocalDb = TestLocalDb.CreateTemp($"sf-resync-{tag}");
        var sfStorage = new StoreAndForwardStorage(sfLocalDb.Db,
            NullLogger<StoreAndForwardStorage>.Instance);
        await sfStorage.InitializeAsync();
        var siteLocalDb = TestLocalDb.CreateTemp($"site-resync-{tag}");
        var siteStorage = new SiteStorageService(siteLocalDb.Db,
            NullLogger<SiteStorageService>.Instance);
        var replicationService = new ReplicationService(
            new StoreAndForwardOptions(), NullLogger<ReplicationService>.Instance);
        // Name MUST be "site-replication" — SendToPeer targets /user/site-replication.
        var actor = node.ActorOf(Props.Create(() => new SiteReplicationActor(
            siteStorage, sfStorage, replicationService, "site-int",
            NullLogger<SiteReplicationActor>.Instance, null, null, null, null)),
            "site-replication");
        return (sfStorage, actor, sfLocalDb, siteLocalDb);
    }

    private static StoreAndForwardMessage NewMessage(string id) => new()
    {
        Id = id,
        Category = StoreAndForwardCategory.Notification,
        Target = "central",
        PayloadJson = "{}",
        CreatedAt = DateTimeOffset.UtcNow,
        Status = StoreAndForwardMessageStatus.Pending,
        MaxRetries = 0,
    };

    private static async Task AwaitAsync(Func<Task<bool>> condition, TimeSpan timeout, string why)
    {
        var deadline = DateTime.UtcNow + timeout;
        while (DateTime.UtcNow < deadline)
        {
            if (await condition()) return;
            await Task.Delay(250);
        }
        throw new TimeoutException(why);
    }
}