feat(localdb)!: replicate site config + sf_messages via CDC, delete the bespoke replicators
Tasks 14, 15 and 16, landed as ONE commit. PLAN DEFECT: these three tasks cannot compile separately. SiteReplicationActor takes a ReplicationService and calls ReplaceAllAsync (Task 14 deletes both); DeploymentManagerActor Tells message types declared in ReplicationMessages.cs (Task 15 deletes it); AkkaHostedService constructs the actor (Task 16). Any ordering leaves a broken intermediate. Combining them also strengthens the invariant Task 14 already stated for itself — the two mechanisms never both run, and never neither. Registered 8 tables in SiteLocalDbSetup.OnReady: sf_messages plus the 7 site config tables. notification_lists and smtp_configurations are deliberately NOT registered — permanently empty by design, so registering them would open a standing replication channel whose only historical payload was plaintext SMTP passwords. Migrate stays the LAST call in OnReady, after all registrations, so migrated rows enter the oplog through live capture triggers. Deleted: SiteReplicationActor, ReplicationMessages.cs, ReplicationService, StoreAndForwardStorage.ReplaceAllAsync, and 6 test files. ReplaceAllAsync is not merely unused but unsafe to keep: a mass DELETE on a now-replicated table would be captured and shipped to the peer. Kept ActiveNodeEvaluator (delivery gate + heartbeat still need it) with its doc corrected, and activeNodeCheck in AkkaHostedService (SiteCommunicationActor). The positional-argument hazard the plan flagged was real: removing DeploymentManagerActor's optional IActorRef? replicationActor shifted 6 trailing optionals, and 4 test call sites bound the wrong arguments with no compile error at some positions. Converted them to named arguments where possible — Props.Create builds an expression tree, which rejects out-of-position named args, so the rest are padded positionally with a comment saying why. The Task 7 'not yet registered' test was INVERTED rather than deleted, and is exact in both directions: too few means a table silently stops replicating, too many means the SMTP tables leak. Added a separate security-named test for those two, and a composite-PK test (LWW keys on the full PK, so a truncated key set would collapse distinct rows). The convergence suites now get their registrations from the real OnReady — their temporary harness registration is deleted, so they prove the cutover rather than agreeing with themselves. Verified: build 0 warnings; SiteRuntime 512, StoreAndForward 130, Host 330, AuditLog 355, ExternalSystemGateway 142, HealthMonitoring 97, LocalDb integration 16 — all pass, 0 failures. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
@@ -45,7 +45,6 @@ public class CachedCallAttemptEmissionTests : IAsyncLifetime, IDisposable
|
||||
_storage,
|
||||
_options,
|
||||
NullLogger<StoreAndForwardService>.Instance,
|
||||
replication: null,
|
||||
cachedCallObserver: _observer,
|
||||
siteId: "site-77");
|
||||
}
|
||||
@@ -490,7 +489,6 @@ public class CachedCallAttemptEmissionTests : IAsyncLifetime, IDisposable
|
||||
RetryTimerInterval = TimeSpan.FromHours(1), // timer never fires in-test
|
||||
},
|
||||
NullLogger<StoreAndForwardService>.Instance,
|
||||
replication: null,
|
||||
cachedCallObserver: observer,
|
||||
siteId: "site-77");
|
||||
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Tests for async replication to standby.
|
||||
/// </summary>
|
||||
public class ReplicationServiceTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly TestLocalDb _localDb;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly ReplicationService _replicationService;
|
||||
|
||||
public ReplicationServiceTests()
|
||||
{
|
||||
_localDb = TestLocalDb.CreateTemp("RepTests");
|
||||
|
||||
_storage = new StoreAndForwardStorage(_localDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
var options = new StoreAndForwardOptions { ReplicationEnabled = true };
|
||||
_replicationService = new ReplicationService(
|
||||
options, NullLogger<ReplicationService>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicateEnqueue_NoHandler_DoesNotThrow()
|
||||
{
|
||||
var msg = CreateMessage("rep1");
|
||||
_replicationService.ReplicateEnqueue(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicateEnqueue_WithHandler_ForwardsOperation()
|
||||
{
|
||||
ReplicationOperation? captured = null;
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
captured = op;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
var msg = CreateMessage("rep2");
|
||||
_replicationService.ReplicateEnqueue(msg);
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(ReplicationOperationType.Add, captured!.OperationType);
|
||||
Assert.Equal("rep2", captured.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicateRemove_WithHandler_ForwardsRemoveOperation()
|
||||
{
|
||||
ReplicationOperation? captured = null;
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
captured = op;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
_replicationService.ReplicateRemove("rep3");
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(ReplicationOperationType.Remove, captured!.OperationType);
|
||||
Assert.Equal("rep3", captured.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicatePark_WithHandler_ForwardsParkOperation()
|
||||
{
|
||||
ReplicationOperation? captured = null;
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
captured = op;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
var msg = CreateMessage("rep4");
|
||||
_replicationService.ReplicatePark(msg);
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(ReplicationOperationType.Park, captured!.OperationType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperationAsync_Add_EnqueuesMessage()
|
||||
{
|
||||
var msg = CreateMessage("apply1");
|
||||
var operation = new ReplicationOperation(ReplicationOperationType.Add, "apply1", msg);
|
||||
|
||||
await _replicationService.ApplyReplicatedOperationAsync(operation, _storage);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("apply1");
|
||||
Assert.NotNull(retrieved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperationAsync_Remove_DeletesMessage()
|
||||
{
|
||||
var msg = CreateMessage("apply2");
|
||||
await _storage.EnqueueAsync(msg);
|
||||
|
||||
var operation = new ReplicationOperation(ReplicationOperationType.Remove, "apply2", null);
|
||||
await _replicationService.ApplyReplicatedOperationAsync(operation, _storage);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("apply2");
|
||||
Assert.Null(retrieved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperationAsync_Park_UpdatesStatus()
|
||||
{
|
||||
var msg = CreateMessage("apply3");
|
||||
await _storage.EnqueueAsync(msg);
|
||||
|
||||
var operation = new ReplicationOperation(ReplicationOperationType.Park, "apply3", msg);
|
||||
await _replicationService.ApplyReplicatedOperationAsync(operation, _storage);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("apply3");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, retrieved!.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicateEnqueue_WhenReplicationDisabled_DoesNothing()
|
||||
{
|
||||
var options = new StoreAndForwardOptions { ReplicationEnabled = false };
|
||||
var service = new ReplicationService(options, NullLogger<ReplicationService>.Instance);
|
||||
|
||||
bool handlerCalled = false;
|
||||
service.SetReplicationHandler(_ => { handlerCalled = true; return Task.CompletedTask; });
|
||||
|
||||
service.ReplicateEnqueue(CreateMessage("disabled1"));
|
||||
|
||||
Assert.False(handlerCalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicateEnqueue_HandlerThrows_DoesNotPropagateException()
|
||||
{
|
||||
_replicationService.SetReplicationHandler(_ =>
|
||||
throw new InvalidOperationException("standby down"));
|
||||
|
||||
_replicationService.ReplicateEnqueue(CreateMessage("err1"));
|
||||
|
||||
await Task.Delay(200);
|
||||
// No exception -- fire-and-forget, best-effort
|
||||
}
|
||||
|
||||
// ── Task 10 (arch review 02): ordered, observable replication dispatch ──
|
||||
|
||||
[Fact]
|
||||
public void ReplicationOperations_AreDispatchedInIssueOrder()
|
||||
{
|
||||
var seen = new List<(ReplicationOperationType, string)>();
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
seen.Add((op.OperationType, op.MessageId));
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
for (var i = 0; i < 200; i++)
|
||||
{
|
||||
_replicationService.ReplicateEnqueue(CreateMessage($"m{i}"));
|
||||
_replicationService.ReplicateRemove($"m{i}");
|
||||
}
|
||||
|
||||
// Inline dispatch: by the time the calls return, every op was handed to the
|
||||
// handler, Add strictly before Remove per id. Pre-fix (Task.Run) this was
|
||||
// racy in both count and order.
|
||||
Assert.Equal(400, seen.Count);
|
||||
for (var i = 0; i < 200; i++)
|
||||
{
|
||||
Assert.Equal((ReplicationOperationType.Add, $"m{i}"), seen[2 * i]);
|
||||
Assert.Equal((ReplicationOperationType.Remove, $"m{i}"), seen[2 * i + 1]);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicationHandlerThrow_IsSwallowed_AndLoggedAtWarning()
|
||||
{
|
||||
var logger = new CapturingLogger<ReplicationService>();
|
||||
var service = new ReplicationService(new StoreAndForwardOptions(), logger);
|
||||
service.SetReplicationHandler(_ => throw new InvalidOperationException("peer gone"));
|
||||
|
||||
service.ReplicateRemove("m1"); // must not throw
|
||||
|
||||
Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning);
|
||||
}
|
||||
|
||||
private static StoreAndForwardMessage CreateMessage(string id)
|
||||
{
|
||||
return new StoreAndForwardMessage
|
||||
{
|
||||
Id = id,
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "target",
|
||||
PayloadJson = "{}",
|
||||
RetryCount = 0,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Minimal in-memory logger that records level + rendered message.</summary>
|
||||
private sealed class CapturingLogger<T> : ILogger<T>
|
||||
{
|
||||
public List<(LogLevel Level, string Message)> Entries { get; } = new();
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
=> Entries.Add((logLevel, formatter(state, exception)));
|
||||
}
|
||||
}
|
||||
-82
@@ -1,82 +0,0 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Characterization pin for the intra-cluster S&F replication contract. A
|
||||
/// <see cref="ReplicationOperation"/> (carrying a full <see cref="StoreAndForwardMessage"/>)
|
||||
/// is Told from the active node to the standby node's replication actor and rides
|
||||
/// the Akka default reflective-JSON wire format. Nothing tested that it round-trips
|
||||
/// or that its type identity is stable — a rename/move, a dropped setter, or a
|
||||
/// non-default-constructible message would silently break standby buffer sync and
|
||||
/// only surface as a divergent buffer after a failover.
|
||||
///
|
||||
/// The serializer swap (proto / explicit bindings) is deferred; until then this pin
|
||||
/// is the guardrail.
|
||||
/// </summary>
|
||||
public class ReplicationWireSerializationPinTests : TestKit
|
||||
{
|
||||
private T RoundTrip<T>(T message)
|
||||
{
|
||||
var serialization = Sys.Serialization;
|
||||
var serializer = serialization.FindSerializerFor(message);
|
||||
var bytes = serializer.ToBinary(message);
|
||||
return (T)serialization.Deserialize(bytes, serializer.Identifier, message!.GetType());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicationOperation_WithFullMessage_RoundTripsOnTheWire()
|
||||
{
|
||||
var message = new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "Operators",
|
||||
PayloadJson = "{\"notificationId\":\"abc\"}",
|
||||
RetryCount = 4,
|
||||
MaxRetries = 0,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
LastAttemptAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
LastError = "central rejected",
|
||||
OriginInstanceName = "Plant.Pump3",
|
||||
ExecutionId = Guid.NewGuid(),
|
||||
SourceScript = "ScriptActor:MonitorSpeed",
|
||||
ParentExecutionId = Guid.NewGuid(),
|
||||
};
|
||||
var original = new ReplicationOperation(ReplicationOperationType.Park, message.Id, message);
|
||||
|
||||
var back = RoundTrip(original);
|
||||
|
||||
Assert.Equal(original.OperationType, back.OperationType);
|
||||
Assert.Equal(original.MessageId, back.MessageId);
|
||||
Assert.NotNull(back.Message);
|
||||
var m = back.Message!;
|
||||
Assert.Equal(message.Id, m.Id);
|
||||
Assert.Equal(message.Category, m.Category);
|
||||
Assert.Equal(message.Target, m.Target);
|
||||
Assert.Equal(message.PayloadJson, m.PayloadJson);
|
||||
Assert.Equal(message.RetryCount, m.RetryCount);
|
||||
Assert.Equal(message.MaxRetries, m.MaxRetries);
|
||||
Assert.Equal(message.RetryIntervalMs, m.RetryIntervalMs);
|
||||
Assert.Equal(message.CreatedAt, m.CreatedAt);
|
||||
Assert.Equal(message.LastAttemptAt, m.LastAttemptAt);
|
||||
Assert.Equal(message.Status, m.Status);
|
||||
Assert.Equal(message.LastError, m.LastError);
|
||||
Assert.Equal(message.OriginInstanceName, m.OriginInstanceName);
|
||||
Assert.Equal(message.ExecutionId, m.ExecutionId);
|
||||
Assert.Equal(message.SourceScript, m.SourceScript);
|
||||
Assert.Equal(message.ParentExecutionId, m.ParentExecutionId);
|
||||
}
|
||||
|
||||
// Type-identity pins: the reflective-JSON wire embeds CLR type manifests, so a
|
||||
// rename/move of either type silently breaks standby replication across a
|
||||
// rolling upgrade. If one fails, you are making a wire-breaking change.
|
||||
[Theory]
|
||||
[InlineData(typeof(ReplicationOperation), "ZB.MOM.WW.ScadaBridge.StoreAndForward.ReplicationOperation")]
|
||||
[InlineData(typeof(StoreAndForwardMessage), "ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardMessage")]
|
||||
public void ReplicationContract_TypeIdentity_IsPinned(Type type, string expectedFullName) =>
|
||||
Assert.Equal(expectedFullName, type.FullName);
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-001: the active node must forward every buffer operation
|
||||
/// (add / remove / park) to the standby via the ReplicationService, so a
|
||||
/// failover does not lose the buffer.
|
||||
/// </summary>
|
||||
public class StoreAndForwardReplicationTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly TestLocalDb _localDb;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
private readonly List<ReplicationOperation> _replicated = new();
|
||||
|
||||
public StoreAndForwardReplicationTests()
|
||||
{
|
||||
_localDb = TestLocalDb.CreateTemp("ReplTests");
|
||||
|
||||
_storage = new StoreAndForwardStorage(_localDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 1,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
ReplicationEnabled = true,
|
||||
};
|
||||
|
||||
var replication = new ReplicationService(options, NullLogger<ReplicationService>.Instance);
|
||||
replication.SetReplicationHandler(op =>
|
||||
{
|
||||
lock (_replicated) _replicated.Add(op);
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, options, NullLogger<StoreAndForwardService>.Instance, replication);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
public void Dispose()
|
||||
{
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
/// <summary>Replication is fire-and-forget (Task.Run); poll until the expected ops arrive.</summary>
|
||||
private async Task<List<ReplicationOperation>> WaitForReplicationAsync(int count)
|
||||
{
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
lock (_replicated)
|
||||
if (_replicated.Count >= count) return _replicated.ToList();
|
||||
await Task.Delay(20);
|
||||
}
|
||||
lock (_replicated) return _replicated.ToList();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BufferingAMessage_ReplicatesAnAddOperation()
|
||||
{
|
||||
// No handler registered → message is buffered → an Add is replicated.
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
var ops = await WaitForReplicationAsync(1);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Add && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SuccessfulRetry_ReplicatesARemoveOperation()
|
||||
{
|
||||
var calls = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => ++calls == 1
|
||||
? throw new HttpRequestException("transient")
|
||||
: Task.FromResult(true));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var ops = await WaitForReplicationAsync(2);
|
||||
Assert.Contains(ops, o => o.OperationType == ReplicationOperationType.Add);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Remove && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ParkedMessage_ReplicatesAParkOperation()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var ops = await WaitForReplicationAsync(2);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Park && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-016: an operator discarding a parked message must replicate
|
||||
/// a Remove so the standby's copy is also deleted (otherwise the discarded
|
||||
/// message reappears in the parked list after a failover).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DiscardingAParkedMessage_ReplicatesARemoveOperation()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync(); // -> parked
|
||||
await WaitForReplicationAsync(2);
|
||||
|
||||
var discarded = await _service.DiscardParkedMessageAsync(result.MessageId);
|
||||
Assert.True(discarded);
|
||||
|
||||
var ops = await WaitForReplicationAsync(3);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Remove && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-016: an operator retrying a parked message must replicate a
|
||||
/// Requeue so the standby's copy moves back to Pending (otherwise it stays
|
||||
/// Parked on the standby and the operator's retry is lost across a failover).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RetryingAParkedMessage_ReplicatesARequeueOperation()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync(); // -> parked
|
||||
await WaitForReplicationAsync(2);
|
||||
|
||||
var retried = await _service.RetryParkedMessageAsync(result.MessageId);
|
||||
Assert.True(retried);
|
||||
|
||||
var ops = await WaitForReplicationAsync(3);
|
||||
var requeue = ops.SingleOrDefault(o =>
|
||||
o.OperationType == ReplicationOperationType.Requeue && o.MessageId == result.MessageId);
|
||||
Assert.NotNull(requeue);
|
||||
Assert.NotNull(requeue!.Message);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, requeue.Message!.Status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-016: the standby applies a Requeue by moving its row back to
|
||||
/// Pending with retry_count = 0, mirroring the active node's local state.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperation_Requeue_MovesStandbyRowBackToPending()
|
||||
{
|
||||
var replication = new ReplicationService(
|
||||
new StoreAndForwardOptions { ReplicationEnabled = true },
|
||||
NullLogger<ReplicationService>.Instance);
|
||||
|
||||
var parked = new StoreAndForwardMessage
|
||||
{
|
||||
Id = "requeue1",
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api",
|
||||
PayloadJson = "{}",
|
||||
RetryCount = 5,
|
||||
MaxRetries = 1,
|
||||
RetryIntervalMs = 0,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
};
|
||||
await _storage.EnqueueAsync(parked);
|
||||
|
||||
var requeued = new StoreAndForwardMessage
|
||||
{
|
||||
Id = parked.Id,
|
||||
Category = parked.Category,
|
||||
Target = parked.Target,
|
||||
PayloadJson = parked.PayloadJson,
|
||||
RetryCount = 0,
|
||||
MaxRetries = parked.MaxRetries,
|
||||
RetryIntervalMs = parked.RetryIntervalMs,
|
||||
CreatedAt = parked.CreatedAt,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
};
|
||||
await replication.ApplyReplicatedOperationAsync(
|
||||
new ReplicationOperation(ReplicationOperationType.Requeue, parked.Id, requeued),
|
||||
_storage);
|
||||
|
||||
var row = await _storage.GetMessageByIdAsync(parked.Id);
|
||||
Assert.NotNull(row);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, row!.Status);
|
||||
Assert.Equal(0, row.RetryCount);
|
||||
}
|
||||
|
||||
private static ReplicationService NewReplicationService() =>
|
||||
new(new StoreAndForwardOptions { ReplicationEnabled = true },
|
||||
NullLogger<ReplicationService>.Instance);
|
||||
|
||||
private static StoreAndForwardMessage NewMessage(string id) => new()
|
||||
{
|
||||
Id = id,
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api",
|
||||
PayloadJson = "{}",
|
||||
RetryCount = 0,
|
||||
MaxRetries = 1,
|
||||
RetryIntervalMs = 0,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// A Park (or Requeue) whose original Add was lost — e.g. the Add's
|
||||
/// fire-and-forget replication dropped — must still materialise the row on the
|
||||
/// standby. The full message rides in the operation, so an upsert self-heals;
|
||||
/// a blind UPDATE would affect 0 rows and the row would be gone forever.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedPark_WhenAddWasLost_MaterializesTheParkedRow()
|
||||
{
|
||||
var service = NewReplicationService();
|
||||
var msg = NewMessage("lost-add-1"); // never Added on this (standby) storage
|
||||
msg.Status = StoreAndForwardMessageStatus.Parked;
|
||||
|
||||
await service.ApplyReplicatedOperationAsync(
|
||||
new ReplicationOperation(ReplicationOperationType.Park, msg.Id, msg), _storage);
|
||||
|
||||
var row = await _storage.GetMessageByIdAsync("lost-add-1");
|
||||
Assert.NotNull(row); // pre-fix: blind UPDATE affected 0 rows, row is gone forever
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, row!.Status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A duplicate Add (e.g. after Task 21's anti-entropy resync re-issues an Add
|
||||
/// the standby already holds) must not violate the PK — the upsert applies
|
||||
/// newest-wins instead of throwing.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedAdd_Twice_IsIdempotent_NewestWins()
|
||||
{
|
||||
var service = NewReplicationService();
|
||||
var msg = NewMessage("dup-add");
|
||||
await service.ApplyReplicatedOperationAsync(
|
||||
new ReplicationOperation(ReplicationOperationType.Add, msg.Id, msg), _storage);
|
||||
msg.RetryCount = 3;
|
||||
await service.ApplyReplicatedOperationAsync( // pre-fix: SqliteException PK violation
|
||||
new ReplicationOperation(ReplicationOperationType.Add, msg.Id, msg), _storage);
|
||||
|
||||
Assert.Equal(3, (await _storage.GetMessageByIdAsync("dup-add"))!.RetryCount);
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,7 @@ public class StoreAndForwardSiteEventTests : IAsyncLifetime, IDisposable
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, _options, NullLogger<StoreAndForwardService>.Instance,
|
||||
replication: null, cachedCallObserver: null, siteId: "site-a",
|
||||
cachedCallObserver: null, siteId: "site-a",
|
||||
siteEventLogger: _siteLog);
|
||||
}
|
||||
|
||||
|
||||
@@ -769,16 +769,15 @@ public class StoreAndForwardStorageTests : IAsyncLifetime, IDisposable
|
||||
Assert.True(truncated); // a third row exists beyond the limit
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplaceAll_SwapsTheEntireBuffer_Atomically()
|
||||
{
|
||||
await _storage.EnqueueAsync(NewMsg("stale"));
|
||||
await _storage.ReplaceAllAsync(new[] { NewMsg("fresh-1"), NewMsg("fresh-2") });
|
||||
|
||||
Assert.Null(await _storage.GetMessageByIdAsync("stale"));
|
||||
Assert.NotNull(await _storage.GetMessageByIdAsync("fresh-1"));
|
||||
Assert.NotNull(await _storage.GetMessageByIdAsync("fresh-2"));
|
||||
}
|
||||
// ReplaceAll_SwapsTheEntireBuffer_Atomically was DELETED with ReplaceAllAsync in
|
||||
// LocalDb Phase 2, and deliberately not replaced. It asserted a destructive
|
||||
// delete-all-then-insert-all, which was the standby's anti-entropy apply: a resync
|
||||
// overwrote the standby's copy with the active node's snapshot. sf_messages is now a
|
||||
// replicated table, so a mass DELETE would be CAPTURED and shipped to the peer — the
|
||||
// method is not merely unused, it is unsafe to keep. LocalDb's own snapshot resync
|
||||
// merges per row under last-writer-wins and never deletes, which is what
|
||||
// LocalDbConfigConvergenceTests.ANodeWithNewerLocalRows_KeepsThem_WhenAPeerSnapshotArrives
|
||||
// now covers.
|
||||
|
||||
// ── Task 23: oldest-parked-age health signal ──
|
||||
|
||||
|
||||
Reference in New Issue
Block a user