using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests; /// /// Phase 2 convergence: the store-and-forward buffer across a real site pair. /// /// /// /// These are specifications, written before the deletion they justify. The bespoke /// ReplicationService (an explicit Add/Remove/Park/Requeue operation stream over Akka) /// is deleted at Task 14 and replaced by LocalDb's trigger-based change capture. Each test /// here is one behaviour the old mechanism provided, restated as an outcome the replacement /// must still deliver — so the cutover has something to be judged against other than "it /// compiles". /// /// /// They are written in terms of rows, not operations, which is the whole point: /// under CDC there is no Add or Park message to observe, only a row that must end up in the /// right state on both nodes. /// /// /// One intent is deliberately not ported: /// ReplicationServiceTests.ReplicationOperations_AreDispatchedInIssueOrder — 200 /// interleaved operations dispatched synchronously, observed in strict issue order. That /// asserts the mechanism (inline fire-and-forget dispatch), not an outcome, and CDC /// capture is asynchronous and batched by construction, so no honest port exists. Its /// portable content is the ordering outcome: an add followed by a remove must never /// converge to present. That is /// . This paragraph exists so a /// future reader does not conclude the test was dropped by accident. /// /// [Collection("LocalDbSitePairConvergence")] public sealed class LocalDbStoreAndForwardConvergenceTests : LocalDbSitePairHarness { // ---- data helpers ----------------------------------------------------------------- /// Buffers a message, as StoreAndForwardStorage.EnqueueAsync does. private static Task EnqueueAsync(ILocalDb db, string id, string target = "ERP.GetOrder") => db.ExecuteAsync( """ INSERT INTO sf_messages ( id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, status) VALUES (@id, 0, @target, '{"order":1}', 0, 50, 30000, @now, @pending) ON CONFLICT(id) DO UPDATE SET status = excluded.status, retry_count = excluded.retry_count; """, new { id, target, now = DateTime.UtcNow.ToString("o"), pending = (int)StoreAndForwardMessageStatus.Pending, }); /// Deletes a delivered message, as the successful-retry path does. private static Task DeleteAsync(ILocalDb db, string id) => db.ExecuteAsync("DELETE FROM sf_messages WHERE id = @id;", new { id }); private static Task SetStatusAsync(ILocalDb db, string id, StoreAndForwardMessageStatus status, int retryCount) => db.ExecuteAsync( "UPDATE sf_messages SET status = @status, retry_count = @retryCount WHERE id = @id;", new { id, status = (int)status, retryCount }); private static async Task<(int Status, int RetryCount)?> ReadAsync(ILocalDb db, string id) { var rows = await db.QueryAsync( "SELECT status, retry_count FROM sf_messages WHERE id = @id", static r => (r.GetInt32(0), r.GetInt32(1)), new { id }); return rows.Count == 0 ? null : rows[0]; } private static async Task ExistsAsync(ILocalDb db, string id) => await ReadAsync(db, id) is not null; private static async Task HasStatusAsync( ILocalDb db, string id, StoreAndForwardMessageStatus status) => await ReadAsync(db, id) is { } row && row.Status == (int)status; // ---- scenarios -------------------------------------------------------------------- [Fact] public async Task BufferedMessage_MaterialisesOnThePeer() { // Was: BufferingAMessage_ReplicatesAnAddOperation. await EnqueueAsync(A, "msg-add"); await WaitUntilAsync( () => ExistsAsync(B, "msg-add"), "a message buffered on node A to appear on node B"); } [Fact] public async Task DeliveredMessage_DisappearsFromThePeer() { // Was: SuccessfulRetry_ReplicatesARemoveOperation. Row deletes propagate as // tombstones under CDC — the peer must not keep re-delivering a message that node A // already succeeded on. await EnqueueAsync(A, "msg-remove"); await WaitUntilAsync(() => ExistsAsync(B, "msg-remove"), "the message to arrive on B first"); await DeleteAsync(A, "msg-remove"); await WaitUntilAsync( async () => !await ExistsAsync(B, "msg-remove"), "the delivered message to be gone from node B"); } [Fact] public async Task ParkedMessage_ShowsAsParkedOnThePeer() { // Was: ParkedMessage_ReplicatesAParkOperation. await EnqueueAsync(A, "msg-park"); await WaitUntilAsync(() => ExistsAsync(B, "msg-park"), "the message to arrive on B first"); await SetStatusAsync(A, "msg-park", StoreAndForwardMessageStatus.Parked, retryCount: 50); await WaitUntilAsync( () => HasStatusAsync(B, "msg-park", StoreAndForwardMessageStatus.Parked), "the parked status to reach node B"); } [Fact] public async Task RequeuedMessage_ResetsStatusAndRetryCountOnThePeer() { // Was: RetryingAParkedMessage_ReplicatesARequeueOperation + // ApplyReplicatedOperation_Requeue_MovesStandbyRowBackToPending. RetryCount matters // as much as status: a peer that took Pending but kept retry_count at max would park // the message again on its first attempt after a failover. await EnqueueAsync(A, "msg-requeue"); await SetStatusAsync(A, "msg-requeue", StoreAndForwardMessageStatus.Parked, retryCount: 50); await WaitUntilAsync( () => HasStatusAsync(B, "msg-requeue", StoreAndForwardMessageStatus.Parked), "the message to be parked on B before it is requeued"); await SetStatusAsync(A, "msg-requeue", StoreAndForwardMessageStatus.Pending, retryCount: 0); await WaitUntilAsync( async () => await ReadAsync(B, "msg-requeue") is { Status: (int)StoreAndForwardMessageStatus.Pending, RetryCount: 0 }, "node B to show the requeued message as Pending with retry_count 0"); } [Fact] public async Task MessageAddedThenRemoved_NeverConvergesToPresent() { // The portable intent of ReplicationOperations_AreDispatchedInIssueOrder (see the // class remarks). Ordering only ever mattered because a remove overtaken by its own // add would resurrect a delivered message and send it twice. Under LWW that // reordering is not preventable by dispatch discipline — it is prevented because the // tombstone carries the later HLC and therefore wins regardless of arrival order. // // Asserting the endpoint rather than the sequence is what makes this portable: the // old test would fail on any async transport even when the outcome was correct. await EnqueueAsync(A, "msg-ordering"); await DeleteAsync(A, "msg-ordering"); // A control row written in the same window. Without it this test is VACUOUS: an // absent row is also what a pair that replicates nothing at all looks like, so it // would pass with capture switched off entirely (observed — it was the only one of // these seven that survived unregistering sf_messages). The control converging is // what makes the absence of msg-ordering evidence rather than silence. await EnqueueAsync(A, "msg-ordering-control"); await WaitUntilAsync( () => ExistsAsync(B, "msg-ordering-control"), "the control message to reach node B, proving the pair was replicating"); await WaitUntilAsync( async () => !await ExistsAsync(B, "msg-ordering") && !await ExistsAsync(A, "msg-ordering"), "the add-then-remove pair to settle as absent on both nodes"); // Absence has to persist, not just occur once: a late-arriving add would flip the row // back and the poll above could have observed a gap it never actually converged to. await Task.Delay(TimeSpan.FromSeconds(2)); Assert.False(await ExistsAsync(B, "msg-ordering")); Assert.False(await ExistsAsync(A, "msg-ordering")); Assert.True(await ExistsAsync(B, "msg-ordering-control")); } [Fact] public async Task SameMessage_BufferedTwice_ConvergesToTheNewerState() { // Was: ApplyReplicatedAdd_Twice_IsIdempotent_NewestWins. Under CDC this is not a // property the application has to implement — LWW on the primary key gives it — but // it is still a property the pair must HAVE, so it is asserted rather than assumed. await EnqueueAsync(A, "msg-twice"); await WaitUntilAsync(() => ExistsAsync(B, "msg-twice"), "the first write to reach B"); await SetStatusAsync(A, "msg-twice", StoreAndForwardMessageStatus.InFlight, retryCount: 3); await WaitUntilAsync( async () => await ReadAsync(B, "msg-twice") is { Status: (int)StoreAndForwardMessageStatus.InFlight, RetryCount: 3 }, "the newer state to win on node B"); } [Fact] public async Task ParkArrivingWithoutItsAdd_StillMaterialisesTheRow() { // Was: ApplyReplicatedPark_WhenAddWasLost_MaterializesTheParkedRow. The bespoke // replicator needed explicit upsert semantics so a lost Add did not leave a Park with // nothing to update. The CDC equivalent is a peer that was offline for the add and // only ever sees the row in its parked state — it must still end up with the row, // not skip it as an update to something it never had. await StopPassiveAsync(); await EnqueueAsync(A, "msg-park-no-add"); await SetStatusAsync(A, "msg-park-no-add", StoreAndForwardMessageStatus.Parked, retryCount: 50); await RestartPairAsync(); await WaitUntilAsync( () => HasStatusAsync(B, "msg-park-no-add", StoreAndForwardMessageStatus.Parked), "node B to materialise a row it only ever saw in its parked state"); } }