using Akka.Actor; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.StoreAndForward; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.TestSupport; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; /// /// N1 regression (review 02 round 2, Critical): the resync authority must use the same /// oldest-Up predicate as the S&F delivery gate. Divergence scenario = the delivering node /// is OLDEST but not LEADER (leader = lowest address), the exact state a rolling restart of /// the lower-address node produces. Pre-fix the delivering node requests a resync from the /// stale peer and ReplaceAllAsync wipes its live buffer. /// public class SfBufferResyncPredicateTests { [Fact] public async Task OldestButNotLeaderNode_KeepsItsBuffer_AndSeedsTheJoiner() { // Two explicit ports, deliberately assigned so the FIRST-started (oldest, // delivering) node has the HIGHER address → the second node is cluster leader. var p1 = TwoNodeClusterFixture.GetFreeTcpPort(); var p2 = TwoNodeClusterFixture.GetFreeTcpPort(); var (portHigh, portLow) = p1 > p2 ? (p1, p2) : (p2, p1); var fixture = await TwoNodeClusterFixture.StartAsync( role: "site-int", portA: portHigh, portB: portLow); // The S&F stores AND SiteStorageService take an ILocalDb (LocalDb has no in-memory // mode), so each node gets its own temp-file local databases. They are disposed // AFTER the cluster is shut down — the actors hold connections while the systems // are alive — and only then are the files (plus their WAL sidecars) deleted. var localDbs = new List(); try { // Real S&F storage + replication actor per node, production default predicate // (no isActiveOverride) — the exact wiring under test. var (storageOldest, _, sfDbOldest, siteDbOldest) = await CreateReplicationActorAsync(fixture.NodeA, "oldest"); localDbs.Add(sfDbOldest); localDbs.Add(siteDbOldest); var (storageJoiner, _, sfDbJoiner, siteDbJoiner) = await CreateReplicationActorAsync(fixture.NodeB, "joiner"); localDbs.Add(sfDbJoiner); localDbs.Add(siteDbJoiner); // The delivering (oldest) node has a live buffered row the standby never saw. await storageOldest.EnqueueAsync(NewMessage("live-row")); // Trigger peer (re)tracking on both sides: each actor got InitialStateAsSnapshot // in PreStart, but the enqueue raced it — re-deliver via a fresh MemberUp is not // needed; OnPeerTracked already fired on join. The resync exchange is async: // wait until the JOINER holds the row (proves the snapshot flowed oldest→joiner, // the correct direction). Pre-fix this times out (the joiner, as leader, never // requests) AND the oldest node's row is deleted by the stale wipe. await AwaitAsync(async () => await storageJoiner.GetMessageByIdAsync("live-row") != null, TimeSpan.FromSeconds(20), "joiner never received the resync snapshot (resync ran in the wrong direction)"); // And the delivering node's buffer is untouched — the N1 wipe assertion. Assert.NotNull(await storageOldest.GetMessageByIdAsync("live-row")); } finally { await fixture.DisposeAsync(); foreach (var localDb in localDbs) { var path = localDb.Path; localDb.Dispose(); TestLocalDb.DeleteFiles(path); } } } private static async Task<( StoreAndForwardStorage Storage, IActorRef Actor, TestLocalDb SfLocalDb, TestLocalDb SiteLocalDb)> CreateReplicationActorAsync(ActorSystem node, string tag) { var sfLocalDb = TestLocalDb.CreateTemp($"sf-resync-{tag}"); var sfStorage = new StoreAndForwardStorage(sfLocalDb.Db, NullLogger.Instance); await sfStorage.InitializeAsync(); var siteLocalDb = TestLocalDb.CreateTemp($"site-resync-{tag}"); var siteStorage = new SiteStorageService(siteLocalDb.Db, NullLogger.Instance); var replicationService = new ReplicationService( new StoreAndForwardOptions(), NullLogger.Instance); // Name MUST be "site-replication" — SendToPeer targets /user/site-replication. var actor = node.ActorOf(Props.Create(() => new SiteReplicationActor( siteStorage, sfStorage, replicationService, "site-int", NullLogger.Instance, null, null, null, null)), "site-replication"); return (sfStorage, actor, sfLocalDb, siteLocalDb); } private static StoreAndForwardMessage NewMessage(string id) => new() { Id = id, Category = StoreAndForwardCategory.Notification, Target = "central", PayloadJson = "{}", CreatedAt = DateTimeOffset.UtcNow, Status = StoreAndForwardMessageStatus.Pending, MaxRetries = 0, }; private static async Task AwaitAsync(Func> condition, TimeSpan timeout, string why) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (await condition()) return; await Task.Delay(250); } throw new TimeoutException(why); } }