refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)
Solution + 23 src projects + 26 test projects renamed; folders, csproj, namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated. ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated. SQL roles/logins, LDAP domains, CLI command name, and CLI config dir (~/.scadalink → ~/.scadabridge) also renamed. Build green; 5 Host.Tests fail awaiting SQL login rename in next commit. Pre-existing StaleTagMonitor timing flakes unchanged. Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,455 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — M3 Bundle E Tasks E4 + E5: the store-and-forward retry
|
||||
/// loop invokes <see cref="ICachedCallLifecycleObserver"/> after every
|
||||
/// cached-call attempt. The observer is given a
|
||||
/// <see cref="CachedCallAttemptContext"/> derived from the underlying
|
||||
/// <see cref="StoreAndForwardMessage"/>; the audit bridge then materialises
|
||||
/// the right <c>CachedCallTelemetry</c> packet (Attempted on every retry,
|
||||
/// CachedResolve on terminal transitions). Tests run with
|
||||
/// <c>DefaultRetryInterval=Zero</c> so the timer-driven retry sweep is
|
||||
/// short-circuited by directly invoking
|
||||
/// <see cref="StoreAndForwardService.RetryPendingMessagesAsync"/>.
|
||||
/// </summary>
|
||||
public class CachedCallAttemptEmissionTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
private readonly StoreAndForwardOptions _options;
|
||||
private readonly CapturingObserver _observer;
|
||||
|
||||
public CachedCallAttemptEmissionTests()
|
||||
{
|
||||
var dbName = $"E4Tests_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
_options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 3,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
};
|
||||
|
||||
_observer = new CapturingObserver();
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage,
|
||||
_options,
|
||||
NullLogger<StoreAndForwardService>.Instance,
|
||||
replication: null,
|
||||
cachedCallObserver: _observer,
|
||||
siteId: "site-77");
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
public void Dispose() => _keepAlive.Dispose();
|
||||
|
||||
/// <summary>
|
||||
/// Captures every observer notification so tests can assert on the
|
||||
/// emitted lifecycle sequence.
|
||||
/// </summary>
|
||||
private sealed class CapturingObserver : ICachedCallLifecycleObserver
|
||||
{
|
||||
public List<CachedCallAttemptContext> Notifications { get; } = new();
|
||||
public Exception? ThrowOnNotify { get; set; }
|
||||
|
||||
public Task OnAttemptCompletedAsync(CachedCallAttemptContext context, CancellationToken ct = default)
|
||||
{
|
||||
if (ThrowOnNotify != null)
|
||||
{
|
||||
return Task.FromException(ThrowOnNotify);
|
||||
}
|
||||
lock (Notifications)
|
||||
{
|
||||
Notifications.Add(context);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<TrackedOperationId> EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory category, string target, int maxRetries = 3)
|
||||
{
|
||||
// The TrackedOperationId is the S&F message id (Bundle E3 contract).
|
||||
var trackedId = TrackedOperationId.New();
|
||||
await _service.EnqueueAsync(
|
||||
category,
|
||||
target,
|
||||
"""{"payload":"x"}""",
|
||||
originInstanceName: "Plant.Pump42",
|
||||
maxRetries: maxRetries,
|
||||
retryInterval: TimeSpan.Zero,
|
||||
attemptImmediateDelivery: false,
|
||||
messageId: trackedId.ToString());
|
||||
return trackedId;
|
||||
}
|
||||
|
||||
// ── Task E4: per-attempt observer notifications ──
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_FailWithHttp500_EmitsAttemptedTelemetry()
|
||||
{
|
||||
// ExternalSystem cached call buffered, retry sweep encounters a
|
||||
// transient failure on the first attempt.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("HTTP 500 from ERP"));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP", maxRetries: 5);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(trackedId, notification.TrackedOperationId);
|
||||
Assert.Equal("ApiOutbound", notification.Channel);
|
||||
Assert.Equal("ERP", notification.Target);
|
||||
Assert.Equal("site-77", notification.SourceSite);
|
||||
Assert.Equal(CachedCallAttemptOutcome.TransientFailure, notification.Outcome);
|
||||
Assert.Equal(1, notification.RetryCount);
|
||||
Assert.Contains("HTTP 500", notification.LastError);
|
||||
Assert.Equal("Plant.Pump42", notification.SourceInstanceId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_Success_EmitsDeliveredOutcome()
|
||||
{
|
||||
// ExternalSystem cached call buffered, retry sweep delivers the
|
||||
// message successfully on its first attempt.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(trackedId, notification.TrackedOperationId);
|
||||
Assert.Equal(CachedCallAttemptOutcome.Delivered, notification.Outcome);
|
||||
Assert.Null(notification.LastError);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_PermanentFailure_EmitsPermanentFailureOutcome()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(false));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(trackedId, notification.TrackedOperationId);
|
||||
Assert.Equal(CachedCallAttemptOutcome.PermanentFailure, notification.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_CachedDbWrite_EmitsDbOutboundChannel()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.CachedDbWrite,
|
||||
_ => Task.FromResult(true));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.CachedDbWrite, "myDb");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(trackedId, notification.TrackedOperationId);
|
||||
Assert.Equal("DbOutbound", notification.Channel);
|
||||
Assert.Equal("myDb", notification.Target);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_NotificationCategory_NoObserverNotification()
|
||||
{
|
||||
// Notifications are NOT cached calls — they're forwarded to central via
|
||||
// a separate forwarder. The observer must not fire for Notification
|
||||
// category messages.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification,
|
||||
_ => Task.FromResult(true));
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.Notification,
|
||||
"alerts",
|
||||
"""{"subject":"x"}""",
|
||||
originInstanceName: "Plant.Pump42",
|
||||
attemptImmediateDelivery: false);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
Assert.Empty(_observer.Notifications);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_MessageIdNotAGuid_NoObserverNotification()
|
||||
{
|
||||
// Pre-M3 cached calls (no TrackedOperationId threaded in) use a random
|
||||
// GUID-N message id from S&F itself. We should still emit (M3 expects
|
||||
// post-rollout these are tracked) — BUT pre-rollout messages can have
|
||||
// a non-parseable id, in which case the observer is silently skipped
|
||||
// to keep S&F bookkeeping intact.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
"ERP",
|
||||
"""{}""",
|
||||
originInstanceName: "Plant.Pump42",
|
||||
attemptImmediateDelivery: false,
|
||||
messageId: "not-a-valid-guid-id");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
Assert.Empty(_observer.Notifications);
|
||||
}
|
||||
|
||||
// ── Task E5: terminal-state observer notifications ──
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_Delivered_EmitsResolveWithDeliveredStatus()
|
||||
{
|
||||
// A successful retry produces a single Delivered observer notification
|
||||
// — the audit bridge maps this to both an Attempted-Delivered audit row
|
||||
// and the terminal CachedResolve(Delivered) row. The S&F layer fires
|
||||
// ONE notification per attempt and lets the bridge fan out as needed.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(trackedId, notification.TrackedOperationId);
|
||||
Assert.Equal(CachedCallAttemptOutcome.Delivered, notification.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_Parked_OnMaxRetries_EmitsParkedMaxRetries()
|
||||
{
|
||||
// Configure handler to throw transient every time.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("Connection refused"));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP", maxRetries: 2);
|
||||
|
||||
// Two sweeps -> RetryCount climbs to 2 -> parked on the second sweep.
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
Assert.Equal(2, _observer.Notifications.Count);
|
||||
Assert.Equal(CachedCallAttemptOutcome.TransientFailure, _observer.Notifications[0].Outcome);
|
||||
Assert.Equal(CachedCallAttemptOutcome.ParkedMaxRetries, _observer.Notifications[1].Outcome);
|
||||
Assert.Equal(trackedId, _observer.Notifications[1].TrackedOperationId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lifecycle_RetryFail_RetrySucceed_EmitsExpectedSequence()
|
||||
{
|
||||
var calls = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ =>
|
||||
{
|
||||
calls++;
|
||||
if (calls == 1) throw new HttpRequestException("transient");
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP", maxRetries: 5);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
Assert.Equal(2, _observer.Notifications.Count);
|
||||
Assert.Equal(CachedCallAttemptOutcome.TransientFailure, _observer.Notifications[0].Outcome);
|
||||
Assert.Equal(1, _observer.Notifications[0].RetryCount);
|
||||
Assert.Equal(CachedCallAttemptOutcome.Delivered, _observer.Notifications[1].Outcome);
|
||||
Assert.Equal(trackedId, _observer.Notifications[1].TrackedOperationId);
|
||||
}
|
||||
|
||||
// ── Audit Log #23 (ExecutionId Task 4): ExecutionId / SourceScript ──
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_CarriesExecutionIdAndSourceScript_FromBufferedMessage()
|
||||
{
|
||||
// A buffered cached call carries the originating script execution's
|
||||
// ExecutionId + SourceScript. The retry sweep must surface both on the
|
||||
// CachedCallAttemptContext handed to the observer so the audit bridge
|
||||
// can stamp them on the retry-loop cached rows.
|
||||
var executionId = Guid.NewGuid();
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("HTTP 503"));
|
||||
|
||||
var trackedId = TrackedOperationId.New();
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
"ERP",
|
||||
"""{"payload":"x"}""",
|
||||
originInstanceName: "Plant.Pump42",
|
||||
maxRetries: 5,
|
||||
retryInterval: TimeSpan.Zero,
|
||||
attemptImmediateDelivery: false,
|
||||
messageId: trackedId.ToString(),
|
||||
executionId: executionId,
|
||||
sourceScript: "Plant.Pump42/OnTick");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(executionId, notification.ExecutionId);
|
||||
Assert.Equal("Plant.Pump42/OnTick", notification.SourceScript);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_NullExecutionIdAndSourceScript_SurfaceAsNull()
|
||||
{
|
||||
// Back-compat: a row buffered without ExecutionId / SourceScript (legacy
|
||||
// enqueue path) must surface them as null on the context, not throw.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Null(notification.ExecutionId);
|
||||
Assert.Null(notification.SourceScript);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TerminalResolve_CarriesExecutionIdAndSourceScript()
|
||||
{
|
||||
// The terminal Delivered notification must also carry the threaded
|
||||
// provenance so the CachedResolve audit row is correlated.
|
||||
var executionId = Guid.NewGuid();
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.CachedDbWrite,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
var trackedId = TrackedOperationId.New();
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.CachedDbWrite,
|
||||
"myDb",
|
||||
"""{"payload":"x"}""",
|
||||
originInstanceName: "Plant.Tank",
|
||||
maxRetries: 3,
|
||||
retryInterval: TimeSpan.Zero,
|
||||
attemptImmediateDelivery: false,
|
||||
messageId: trackedId.ToString(),
|
||||
executionId: executionId,
|
||||
sourceScript: "Plant.Tank/OnAlarm");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(CachedCallAttemptOutcome.Delivered, notification.Outcome);
|
||||
Assert.Equal(executionId, notification.ExecutionId);
|
||||
Assert.Equal("Plant.Tank/OnAlarm", notification.SourceScript);
|
||||
}
|
||||
|
||||
// ── Audit Log #23 (ParentExecutionId Task 6): ParentExecutionId ──
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_CarriesParentExecutionId_FromBufferedMessage()
|
||||
{
|
||||
// A cached call enqueued from an inbound-API-routed script run carries
|
||||
// the spawning execution's ParentExecutionId. The retry sweep must
|
||||
// surface it on the CachedCallAttemptContext beside ExecutionId so the
|
||||
// audit bridge can stamp it on the retry-loop cached rows.
|
||||
var parentExecutionId = Guid.NewGuid();
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("HTTP 503"));
|
||||
|
||||
var trackedId = TrackedOperationId.New();
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
"ERP",
|
||||
"""{"payload":"x"}""",
|
||||
originInstanceName: "Plant.Pump42",
|
||||
maxRetries: 5,
|
||||
retryInterval: TimeSpan.Zero,
|
||||
attemptImmediateDelivery: false,
|
||||
messageId: trackedId.ToString(),
|
||||
parentExecutionId: parentExecutionId);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(parentExecutionId, notification.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Attempt_NullParentExecutionId_SurfacesAsNull()
|
||||
{
|
||||
// Non-routed run: the originating script was not spawned by an
|
||||
// inbound-API request, so no ParentExecutionId is threaded. It must
|
||||
// surface as null on the context, not throw.
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Null(notification.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TerminalResolve_CarriesParentExecutionId()
|
||||
{
|
||||
// The terminal Delivered notification must also carry the threaded
|
||||
// ParentExecutionId so the CachedResolve audit row correlates back to
|
||||
// the spawning inbound-API execution.
|
||||
var parentExecutionId = Guid.NewGuid();
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.CachedDbWrite,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
var trackedId = TrackedOperationId.New();
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.CachedDbWrite,
|
||||
"myDb",
|
||||
"""{"payload":"x"}""",
|
||||
originInstanceName: "Plant.Tank",
|
||||
maxRetries: 3,
|
||||
retryInterval: TimeSpan.Zero,
|
||||
attemptImmediateDelivery: false,
|
||||
messageId: trackedId.ToString(),
|
||||
parentExecutionId: parentExecutionId);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var notification = Assert.Single(_observer.Notifications);
|
||||
Assert.Equal(CachedCallAttemptOutcome.Delivered, notification.Outcome);
|
||||
Assert.Equal(parentExecutionId, notification.ParentExecutionId);
|
||||
}
|
||||
|
||||
// ── Best-effort contract: observer throws must NOT corrupt retry bookkeeping ──
|
||||
|
||||
[Fact]
|
||||
public async Task Observer_Throws_DoesNotCorruptRetryCount()
|
||||
{
|
||||
_observer.ThrowOnNotify = new InvalidOperationException("simulated audit failure");
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
var trackedId = await EnqueueBufferedAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP");
|
||||
|
||||
// Must not throw — observer is best-effort.
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
// The message was delivered (handler returned true) so it should be gone.
|
||||
var msg = await _storage.GetMessageByIdAsync(trackedId.ToString());
|
||||
Assert.Null(msg);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Notification Outbox: tests for the site Store-and-Forward notification delivery
|
||||
/// handler. "Delivering" a buffered notification means forwarding it to the central
|
||||
/// cluster (via the site communication actor) and treating central's
|
||||
/// <see cref="NotificationSubmitAck"/> as the delivery outcome.
|
||||
/// </summary>
|
||||
public class NotificationForwarderTests : TestKit
|
||||
{
|
||||
private static readonly TimeSpan ForwardTimeout = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a buffered notification S&F message whose payload matches the shape
|
||||
/// produced by the site <c>Notify.Send</c> enqueue path (Task 19): a serialized
|
||||
/// <see cref="NotificationSubmit"/> carrying a script-generated
|
||||
/// <see cref="NotificationSubmit.NotificationId"/>. The S&F message
|
||||
/// <see cref="StoreAndForwardMessage.Id"/> equals that same id.
|
||||
/// </summary>
|
||||
private static StoreAndForwardMessage BufferedNotification(
|
||||
string id = "msg-1", string listName = "Operators",
|
||||
string subject = "Pump alarm", string message = "Pump 3 tripped",
|
||||
string? originInstance = "Plant.Pump3", string? sourceScript = "alarmScript",
|
||||
Guid? originExecutionId = null)
|
||||
{
|
||||
var payload = JsonSerializer.Serialize(new NotificationSubmit(
|
||||
NotificationId: id,
|
||||
ListName: listName,
|
||||
Subject: subject,
|
||||
Body: message,
|
||||
// SourceSiteId is re-stamped by the forwarder; the enqueue side leaves it blank.
|
||||
SourceSiteId: string.Empty,
|
||||
SourceInstanceId: originInstance,
|
||||
SourceScript: sourceScript,
|
||||
SiteEnqueuedAt: DateTimeOffset.UtcNow,
|
||||
OriginExecutionId: originExecutionId));
|
||||
return new StoreAndForwardMessage
|
||||
{
|
||||
Id = id,
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = listName,
|
||||
PayloadJson = payload,
|
||||
OriginInstanceName = originInstance,
|
||||
};
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_ForwardsNotificationSubmitToCentralTarget_AndReturnsTrueOnAccept()
|
||||
{
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
var msg = BufferedNotification(
|
||||
id: "msg-1", listName: "Operators", subject: "Pump alarm",
|
||||
message: "Pump 3 tripped", originInstance: "Plant.Pump3");
|
||||
|
||||
var deliverTask = forwarder.DeliverAsync(msg);
|
||||
|
||||
// The central target receives a NotificationSubmit whose fields map from the
|
||||
// buffered payload; reply Accepted so the handler completes as delivered.
|
||||
var submit = centralProbe.ExpectMsg<NotificationSubmit>();
|
||||
Assert.Equal("msg-1", submit.NotificationId);
|
||||
Assert.Equal("Operators", submit.ListName);
|
||||
Assert.Equal("Pump alarm", submit.Subject);
|
||||
Assert.Equal("Pump 3 tripped", submit.Body);
|
||||
// SourceSiteId is re-stamped by the forwarder from its own site id.
|
||||
Assert.Equal("site-7", submit.SourceSiteId);
|
||||
Assert.Equal("Plant.Pump3", submit.SourceInstanceId);
|
||||
// The originating script travels through from the buffered payload.
|
||||
Assert.Equal("alarmScript", submit.SourceScript);
|
||||
centralProbe.Reply(new NotificationSubmitAck(submit.NotificationId, Accepted: true, Error: null));
|
||||
|
||||
Assert.True(await deliverTask);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_PreservesOriginExecutionId_FromBufferedPayload()
|
||||
{
|
||||
// Audit Log #23: the buffered payload's OriginExecutionId is the per-run
|
||||
// id stamped at Notify.Send time. The forwarder re-stamps only the four
|
||||
// fields it authoritatively owns (NotificationId, ListName, SourceSiteId,
|
||||
// SourceInstanceId) via the `with` expression — OriginExecutionId is
|
||||
// preserved precisely BY being absent from that `with` block. This test
|
||||
// pins that: if OriginExecutionId is ever added to the `with` expression
|
||||
// (e.g. reset to null), the forwarded NotificationSubmit would lose the
|
||||
// per-run id and central could not echo it onto NotifyDeliver rows.
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
var executionId = Guid.NewGuid();
|
||||
var msg = BufferedNotification(id: "msg-exec", originExecutionId: executionId);
|
||||
|
||||
var deliverTask = forwarder.DeliverAsync(msg);
|
||||
|
||||
var submit = centralProbe.ExpectMsg<NotificationSubmit>();
|
||||
Assert.Equal(executionId, submit.OriginExecutionId);
|
||||
centralProbe.Reply(new NotificationSubmitAck(submit.NotificationId, Accepted: true, Error: null));
|
||||
|
||||
Assert.True(await deliverTask);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_FallsBackToTarget_WhenPayloadListNameIsEmpty()
|
||||
{
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
// A buffered payload carrying an empty-string ListName: the empty value must not
|
||||
// be forwarded — the forwarder falls back to the S&F message Target instead.
|
||||
var payload = JsonSerializer.Serialize(new NotificationSubmit(
|
||||
NotificationId: "msg-empty-list",
|
||||
ListName: "",
|
||||
Subject: "Pump alarm",
|
||||
Body: "Pump 3 tripped",
|
||||
SourceSiteId: string.Empty,
|
||||
SourceInstanceId: "Plant.Pump3",
|
||||
SourceScript: null,
|
||||
SiteEnqueuedAt: DateTimeOffset.UtcNow));
|
||||
var msg = new StoreAndForwardMessage
|
||||
{
|
||||
Id = "msg-empty-list",
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "Operators",
|
||||
PayloadJson = payload,
|
||||
OriginInstanceName = "Plant.Pump3",
|
||||
};
|
||||
|
||||
var deliverTask = forwarder.DeliverAsync(msg);
|
||||
|
||||
var submit = centralProbe.ExpectMsg<NotificationSubmit>();
|
||||
Assert.Equal("Operators", submit.ListName);
|
||||
centralProbe.Reply(new NotificationSubmitAck(submit.NotificationId, Accepted: true, Error: null));
|
||||
|
||||
Assert.True(await deliverTask);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_ThrowsTransient_WhenAckIsNotAccepted()
|
||||
{
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
var deliverTask = forwarder.DeliverAsync(BufferedNotification());
|
||||
|
||||
var submit = centralProbe.ExpectMsg<NotificationSubmit>();
|
||||
centralProbe.Reply(new NotificationSubmitAck(
|
||||
submit.NotificationId, Accepted: false, Error: "central rejected"));
|
||||
|
||||
// A non-accepted ack is a transient failure — the handler throws so the S&F
|
||||
// engine keeps the message buffered and retries the forward.
|
||||
await Assert.ThrowsAnyAsync<Exception>(() => deliverTask);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_ThrowsTransient_WhenNoReplyWithinTimeout()
|
||||
{
|
||||
// A probe that never replies stands in for central being unreachable.
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", TimeSpan.FromMilliseconds(300));
|
||||
|
||||
// No reply within the timeout → transient failure → throw.
|
||||
await Assert.ThrowsAnyAsync<Exception>(() => forwarder.DeliverAsync(BufferedNotification()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_UsesStableNotificationId_AcrossRetriesOfSameMessage()
|
||||
{
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
var buffered = BufferedNotification(id: "stable-msg-id");
|
||||
|
||||
var first = forwarder.DeliverAsync(buffered);
|
||||
var submit1 = centralProbe.ExpectMsg<NotificationSubmit>();
|
||||
centralProbe.Reply(new NotificationSubmitAck(submit1.NotificationId, true, null));
|
||||
await first;
|
||||
|
||||
var second = forwarder.DeliverAsync(buffered);
|
||||
var submit2 = centralProbe.ExpectMsg<NotificationSubmit>();
|
||||
centralProbe.Reply(new NotificationSubmitAck(submit2.NotificationId, true, null));
|
||||
await second;
|
||||
|
||||
// The NotificationId is the central idempotency key — it must be identical for
|
||||
// every forward attempt of the same buffered S&F message.
|
||||
Assert.Equal(submit1.NotificationId, submit2.NotificationId);
|
||||
Assert.Equal("stable-msg-id", submit1.NotificationId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_CorruptJsonPayload_ReturnsTrue_AndDoesNotForwardAnything()
|
||||
{
|
||||
// Regression test for StoreAndForward-018. The design doc forbids parking
|
||||
// notifications ("notifications do not park — they are retried at the fixed
|
||||
// forward interval until central acks"; Component-StoreAndForward.md). The
|
||||
// previous implementation returned false on a corrupt payload, which the S&F
|
||||
// engine interprets as a permanent failure and parks the row — contradicting
|
||||
// the invariant. The fix: discard a corrupt buffered notification by
|
||||
// returning true (engine clears the buffer via its normal success path),
|
||||
// with a Warning log line carrying the row id and a payload preview.
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
var corrupt = new StoreAndForwardMessage
|
||||
{
|
||||
Id = "msg-corrupt",
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "Operators",
|
||||
PayloadJson = "{not-valid-json",
|
||||
OriginInstanceName = "Plant.Pump3",
|
||||
};
|
||||
|
||||
Assert.True(await forwarder.DeliverAsync(corrupt));
|
||||
|
||||
// The corrupt-payload path must NOT round-trip to central — no
|
||||
// NotificationSubmit / no Ask. ExpectNoMsg confirms nothing was forwarded.
|
||||
centralProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deliver_NullDeserializedPayload_ReturnsTrue_AndDoesNotForwardAnything()
|
||||
{
|
||||
// The companion case to corrupt JSON: the payload is valid JSON but
|
||||
// deserialises to null (e.g. "null"). Same treatment per StoreAndForward-018
|
||||
// — discard rather than park.
|
||||
var centralProbe = CreateTestProbe();
|
||||
var forwarder = new NotificationForwarder(
|
||||
centralProbe.Ref, "site-7", ForwardTimeout);
|
||||
|
||||
var nullPayload = new StoreAndForwardMessage
|
||||
{
|
||||
Id = "msg-null",
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "Operators",
|
||||
PayloadJson = "null",
|
||||
OriginInstanceName = "Plant.Pump3",
|
||||
};
|
||||
|
||||
Assert.True(await forwarder.DeliverAsync(nullPayload));
|
||||
centralProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-013: tests for the <see cref="ParkedMessageHandlerActor"/> actor
|
||||
/// bridge — the Query/Retry/Discard request-to-response mapping and the
|
||||
/// <c>ExtractMethodName</c> payload-JSON parsing (including the malformed-JSON branch).
|
||||
/// </summary>
|
||||
public class ParkedMessageHandlerActorTests : TestKit, IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
|
||||
public ParkedMessageHandlerActorTests()
|
||||
{
|
||||
var connStr = $"Data Source=ActorTests_{Guid.NewGuid():N};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 1,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
ReplicationEnabled = false,
|
||||
};
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, options, NullLogger<StoreAndForwardService>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing) _keepAlive.Dispose();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
/// <summary>Enqueues a message and parks it via the retry sweep (MaxRetries = 1).</summary>
|
||||
private async Task<string> ParkMessageAsync(StoreAndForwardCategory category, string payloadJson)
|
||||
{
|
||||
_service.RegisterDeliveryHandler(category, _ => throw new HttpRequestException("always fails"));
|
||||
var result = await _service.EnqueueAsync(category, "target", payloadJson, maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
return result.MessageId;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Query_ReturnsParkedEntries_WithExtractedMethodName()
|
||||
{
|
||||
await ParkMessageAsync(StoreAndForwardCategory.ExternalSystem,
|
||||
"""{"MethodName":"StartPump","Args":{}}""");
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageQueryRequest("corr-1", "site-1", 1, 50, DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageQueryResponse>();
|
||||
Assert.True(response.Success);
|
||||
Assert.Equal("corr-1", response.CorrelationId);
|
||||
Assert.Equal("site-1", response.SiteId);
|
||||
Assert.Single(response.Messages);
|
||||
Assert.Equal("StartPump", response.Messages[0].MethodName);
|
||||
Assert.Equal(StoreAndForwardCategory.ExternalSystem, response.Messages[0].Category);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Query_NotificationPayload_UsesSubjectAsMethodName()
|
||||
{
|
||||
await ParkMessageAsync(StoreAndForwardCategory.Notification,
|
||||
"""{"Subject":"Tank overflow","Body":"..."}""");
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageQueryRequest("corr-2", "site-1", 1, 50, DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageQueryResponse>();
|
||||
Assert.True(response.Success);
|
||||
Assert.Equal("Tank overflow", response.Messages[0].MethodName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Query_MalformedJsonPayload_FallsBackToCategoryName()
|
||||
{
|
||||
await ParkMessageAsync(StoreAndForwardCategory.CachedDbWrite, "not-valid-json{");
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageQueryRequest("corr-3", "site-1", 1, 50, DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageQueryResponse>();
|
||||
Assert.True(response.Success);
|
||||
// Malformed JSON must not throw — ExtractMethodName falls back to the category.
|
||||
Assert.Equal(StoreAndForwardCategory.CachedDbWrite.ToString(), response.Messages[0].MethodName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Query_PayloadWithoutMethodNameOrSubject_FallsBackToCategoryName()
|
||||
{
|
||||
await ParkMessageAsync(StoreAndForwardCategory.ExternalSystem, """{"Unrelated":"value"}""");
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageQueryRequest("corr-4", "site-1", 1, 50, DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageQueryResponse>();
|
||||
Assert.Equal(StoreAndForwardCategory.ExternalSystem.ToString(), response.Messages[0].MethodName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_ParkedMessage_ReturnsSuccess()
|
||||
{
|
||||
var messageId = await ParkMessageAsync(StoreAndForwardCategory.ExternalSystem, """{}""");
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageRetryRequest("corr-5", "site-1", messageId, DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageRetryResponse>();
|
||||
Assert.True(response.Success);
|
||||
Assert.Equal("corr-5", response.CorrelationId);
|
||||
Assert.Null(response.ErrorMessage);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(messageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Retry_UnknownMessage_ReturnsFailureWithMessage()
|
||||
{
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageRetryRequest("corr-6", "site-1", "does-not-exist", DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageRetryResponse>();
|
||||
Assert.False(response.Success);
|
||||
Assert.Equal("corr-6", response.CorrelationId);
|
||||
Assert.NotNull(response.ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Discard_ParkedMessage_ReturnsSuccessAndRemovesMessage()
|
||||
{
|
||||
var messageId = await ParkMessageAsync(StoreAndForwardCategory.ExternalSystem, """{}""");
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageDiscardRequest("corr-7", "site-1", messageId, DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageDiscardResponse>();
|
||||
Assert.True(response.Success);
|
||||
Assert.Equal("corr-7", response.CorrelationId);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(messageId);
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Discard_UnknownMessage_ReturnsFailureWithMessage()
|
||||
{
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new ParkedMessageDiscardRequest("corr-8", "site-1", "does-not-exist", DateTimeOffset.UtcNow));
|
||||
|
||||
var response = ExpectMsg<ParkedMessageDiscardResponse>();
|
||||
Assert.False(response.Success);
|
||||
Assert.NotNull(response.ErrorMessage);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Task 5 (#22 Retry/Discard relay): tests the site-side execution of a
|
||||
/// central→site <see cref="RetryParkedOperation"/> / <see cref="DiscardParkedOperation"/>
|
||||
/// relay command on the <see cref="ParkedMessageHandlerActor"/>. The cached
|
||||
/// call's S&F buffer message id is the <see cref="TrackedOperationId"/>, so
|
||||
/// the handler resolves the parked row directly from the tracked id and reuses
|
||||
/// the existing parked-message Retry/Discard primitive. A non-parked operation
|
||||
/// must be a safe no-op (<c>Applied=false</c>), never a corruption.
|
||||
/// </summary>
|
||||
public class ParkedOperationRelayTests : TestKit, IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
|
||||
public ParkedOperationRelayTests()
|
||||
{
|
||||
var connStr = $"Data Source=RelayTests_{Guid.NewGuid():N};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 1,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
ReplicationEnabled = false,
|
||||
};
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, options, NullLogger<StoreAndForwardService>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing) _keepAlive.Dispose();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueues a cached-call message whose S&F id is the supplied
|
||||
/// <see cref="TrackedOperationId"/> and parks it via the retry sweep.
|
||||
/// </summary>
|
||||
private async Task ParkCachedCallAsync(TrackedOperationId id)
|
||||
{
|
||||
_service.RegisterDeliveryHandler(
|
||||
StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("always fails"));
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP.GetOrder", """{}""",
|
||||
maxRetries: 1, messageId: id.ToString());
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryParkedOperation_ParkedCachedCall_ResetsToPendingAndApplied()
|
||||
{
|
||||
var id = TrackedOperationId.New();
|
||||
await ParkCachedCallAsync(id);
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new RetryParkedOperation("corr-1", id));
|
||||
|
||||
var ack = ExpectMsg<ParkedOperationActionAck>();
|
||||
Assert.True(ack.Applied);
|
||||
Assert.Equal("corr-1", ack.CorrelationId);
|
||||
Assert.Null(ack.ErrorMessage);
|
||||
|
||||
// The parked row was reset back to Pending so the retry sweep picks it up.
|
||||
var msg = await _storage.GetMessageByIdAsync(id.ToString());
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscardParkedOperation_ParkedCachedCall_RemovesRowAndApplied()
|
||||
{
|
||||
var id = TrackedOperationId.New();
|
||||
await ParkCachedCallAsync(id);
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new DiscardParkedOperation("corr-2", id));
|
||||
|
||||
var ack = ExpectMsg<ParkedOperationActionAck>();
|
||||
Assert.True(ack.Applied);
|
||||
Assert.Equal("corr-2", ack.CorrelationId);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(id.ToString());
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RetryParkedOperation_UnknownOperation_IsSafeNoOp()
|
||||
{
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new RetryParkedOperation("corr-3", TrackedOperationId.New()));
|
||||
|
||||
var ack = ExpectMsg<ParkedOperationActionAck>();
|
||||
// No parked row matched — definitive "nothing to do", not an error.
|
||||
Assert.False(ack.Applied);
|
||||
Assert.Equal("corr-3", ack.CorrelationId);
|
||||
Assert.Null(ack.ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryParkedOperation_NonParkedOperation_IsSafeNoOpAndDoesNotCorrupt()
|
||||
{
|
||||
// Enqueue a cached call but DO NOT park it — it stays Pending.
|
||||
var id = TrackedOperationId.New();
|
||||
_service.RegisterDeliveryHandler(
|
||||
StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fails"));
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP.GetOrder", """{}""",
|
||||
maxRetries: 5, messageId: id.ToString());
|
||||
|
||||
var before = await _storage.GetMessageByIdAsync(id.ToString());
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, before!.Status);
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new RetryParkedOperation("corr-4", id));
|
||||
|
||||
var ack = ExpectMsg<ParkedOperationActionAck>();
|
||||
// The row is Pending, not Parked — Retry must be a no-op, not a mutation.
|
||||
Assert.False(ack.Applied);
|
||||
|
||||
var after = await _storage.GetMessageByIdAsync(id.ToString());
|
||||
Assert.NotNull(after);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, after!.Status);
|
||||
// retry_count untouched — a Parked-only Retry must not reset a live row.
|
||||
Assert.Equal(before.RetryCount, after.RetryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscardParkedOperation_NonParkedOperation_IsSafeNoOp()
|
||||
{
|
||||
var id = TrackedOperationId.New();
|
||||
_service.RegisterDeliveryHandler(
|
||||
StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fails"));
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "ERP.GetOrder", """{}""",
|
||||
maxRetries: 5, messageId: id.ToString());
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new ParkedMessageHandlerActor(_service, "site-1")));
|
||||
actor.Tell(new DiscardParkedOperation("corr-5", id));
|
||||
|
||||
var ack = ExpectMsg<ParkedOperationActionAck>();
|
||||
Assert.False(ack.Applied);
|
||||
|
||||
// The Pending row must NOT have been deleted by a Parked-only Discard.
|
||||
var after = await _storage.GetMessageByIdAsync(id.ToString());
|
||||
Assert.NotNull(after);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Tests for async replication to standby.
|
||||
/// </summary>
|
||||
public class ReplicationServiceTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly ReplicationService _replicationService;
|
||||
|
||||
public ReplicationServiceTests()
|
||||
{
|
||||
var dbName = $"RepTests_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
var options = new StoreAndForwardOptions { ReplicationEnabled = true };
|
||||
_replicationService = new ReplicationService(
|
||||
options, NullLogger<ReplicationService>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
public void Dispose() => _keepAlive.Dispose();
|
||||
|
||||
[Fact]
|
||||
public void ReplicateEnqueue_NoHandler_DoesNotThrow()
|
||||
{
|
||||
var msg = CreateMessage("rep1");
|
||||
_replicationService.ReplicateEnqueue(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicateEnqueue_WithHandler_ForwardsOperation()
|
||||
{
|
||||
ReplicationOperation? captured = null;
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
captured = op;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
var msg = CreateMessage("rep2");
|
||||
_replicationService.ReplicateEnqueue(msg);
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(ReplicationOperationType.Add, captured!.OperationType);
|
||||
Assert.Equal("rep2", captured.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicateRemove_WithHandler_ForwardsRemoveOperation()
|
||||
{
|
||||
ReplicationOperation? captured = null;
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
captured = op;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
_replicationService.ReplicateRemove("rep3");
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(ReplicationOperationType.Remove, captured!.OperationType);
|
||||
Assert.Equal("rep3", captured.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicatePark_WithHandler_ForwardsParkOperation()
|
||||
{
|
||||
ReplicationOperation? captured = null;
|
||||
_replicationService.SetReplicationHandler(op =>
|
||||
{
|
||||
captured = op;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
var msg = CreateMessage("rep4");
|
||||
_replicationService.ReplicatePark(msg);
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(ReplicationOperationType.Park, captured!.OperationType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperationAsync_Add_EnqueuesMessage()
|
||||
{
|
||||
var msg = CreateMessage("apply1");
|
||||
var operation = new ReplicationOperation(ReplicationOperationType.Add, "apply1", msg);
|
||||
|
||||
await _replicationService.ApplyReplicatedOperationAsync(operation, _storage);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("apply1");
|
||||
Assert.NotNull(retrieved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperationAsync_Remove_DeletesMessage()
|
||||
{
|
||||
var msg = CreateMessage("apply2");
|
||||
await _storage.EnqueueAsync(msg);
|
||||
|
||||
var operation = new ReplicationOperation(ReplicationOperationType.Remove, "apply2", null);
|
||||
await _replicationService.ApplyReplicatedOperationAsync(operation, _storage);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("apply2");
|
||||
Assert.Null(retrieved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperationAsync_Park_UpdatesStatus()
|
||||
{
|
||||
var msg = CreateMessage("apply3");
|
||||
await _storage.EnqueueAsync(msg);
|
||||
|
||||
var operation = new ReplicationOperation(ReplicationOperationType.Park, "apply3", msg);
|
||||
await _replicationService.ApplyReplicatedOperationAsync(operation, _storage);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("apply3");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, retrieved!.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicateEnqueue_WhenReplicationDisabled_DoesNothing()
|
||||
{
|
||||
var options = new StoreAndForwardOptions { ReplicationEnabled = false };
|
||||
var service = new ReplicationService(options, NullLogger<ReplicationService>.Instance);
|
||||
|
||||
bool handlerCalled = false;
|
||||
service.SetReplicationHandler(_ => { handlerCalled = true; return Task.CompletedTask; });
|
||||
|
||||
service.ReplicateEnqueue(CreateMessage("disabled1"));
|
||||
|
||||
Assert.False(handlerCalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicateEnqueue_HandlerThrows_DoesNotPropagateException()
|
||||
{
|
||||
_replicationService.SetReplicationHandler(_ =>
|
||||
throw new InvalidOperationException("standby down"));
|
||||
|
||||
_replicationService.ReplicateEnqueue(CreateMessage("err1"));
|
||||
|
||||
await Task.Delay(200);
|
||||
// No exception -- fire-and-forget, best-effort
|
||||
}
|
||||
|
||||
private static StoreAndForwardMessage CreateMessage(string id)
|
||||
{
|
||||
return new StoreAndForwardMessage
|
||||
{
|
||||
Id = id,
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "target",
|
||||
PayloadJson = "{}",
|
||||
RetryCount = 0,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-9: Tests for StoreAndForwardOptions defaults and configuration.
|
||||
/// </summary>
|
||||
public class StoreAndForwardOptionsTests
|
||||
{
|
||||
[Fact]
|
||||
public void DefaultOptions_HasReasonableDefaults()
|
||||
{
|
||||
var options = new StoreAndForwardOptions();
|
||||
|
||||
Assert.Equal("./data/store-and-forward.db", options.SqliteDbPath);
|
||||
Assert.True(options.ReplicationEnabled);
|
||||
Assert.Equal(TimeSpan.FromSeconds(30), options.DefaultRetryInterval);
|
||||
Assert.Equal(50, options.DefaultMaxRetries);
|
||||
Assert.Equal(TimeSpan.FromSeconds(10), options.RetryTimerInterval);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Options_CanBeCustomized()
|
||||
{
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
SqliteDbPath = "/custom/path.db",
|
||||
ReplicationEnabled = false,
|
||||
DefaultRetryInterval = TimeSpan.FromMinutes(5),
|
||||
DefaultMaxRetries = 100,
|
||||
RetryTimerInterval = TimeSpan.FromSeconds(30)
|
||||
};
|
||||
|
||||
Assert.Equal("/custom/path.db", options.SqliteDbPath);
|
||||
Assert.False(options.ReplicationEnabled);
|
||||
Assert.Equal(TimeSpan.FromMinutes(5), options.DefaultRetryInterval);
|
||||
Assert.Equal(100, options.DefaultMaxRetries);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-001: the active node must forward every buffer operation
|
||||
/// (add / remove / park) to the standby via the ReplicationService, so a
|
||||
/// failover does not lose the buffer.
|
||||
/// </summary>
|
||||
public class StoreAndForwardReplicationTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
private readonly List<ReplicationOperation> _replicated = new();
|
||||
|
||||
public StoreAndForwardReplicationTests()
|
||||
{
|
||||
var connStr = $"Data Source=ReplTests_{Guid.NewGuid():N};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 1,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
ReplicationEnabled = true,
|
||||
};
|
||||
|
||||
var replication = new ReplicationService(options, NullLogger<ReplicationService>.Instance);
|
||||
replication.SetReplicationHandler(op =>
|
||||
{
|
||||
lock (_replicated) _replicated.Add(op);
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, options, NullLogger<StoreAndForwardService>.Instance, replication);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
public void Dispose() => _keepAlive.Dispose();
|
||||
|
||||
/// <summary>Replication is fire-and-forget (Task.Run); poll until the expected ops arrive.</summary>
|
||||
private async Task<List<ReplicationOperation>> WaitForReplicationAsync(int count)
|
||||
{
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
lock (_replicated)
|
||||
if (_replicated.Count >= count) return _replicated.ToList();
|
||||
await Task.Delay(20);
|
||||
}
|
||||
lock (_replicated) return _replicated.ToList();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BufferingAMessage_ReplicatesAnAddOperation()
|
||||
{
|
||||
// No handler registered → message is buffered → an Add is replicated.
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
var ops = await WaitForReplicationAsync(1);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Add && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SuccessfulRetry_ReplicatesARemoveOperation()
|
||||
{
|
||||
var calls = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => ++calls == 1
|
||||
? throw new HttpRequestException("transient")
|
||||
: Task.FromResult(true));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var ops = await WaitForReplicationAsync(2);
|
||||
Assert.Contains(ops, o => o.OperationType == ReplicationOperationType.Add);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Remove && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ParkedMessage_ReplicatesAParkOperation()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var ops = await WaitForReplicationAsync(2);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Park && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-016: an operator discarding a parked message must replicate
|
||||
/// a Remove so the standby's copy is also deleted (otherwise the discarded
|
||||
/// message reappears in the parked list after a failover).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DiscardingAParkedMessage_ReplicatesARemoveOperation()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync(); // -> parked
|
||||
await WaitForReplicationAsync(2);
|
||||
|
||||
var discarded = await _service.DiscardParkedMessageAsync(result.MessageId);
|
||||
Assert.True(discarded);
|
||||
|
||||
var ops = await WaitForReplicationAsync(3);
|
||||
Assert.Contains(ops, o =>
|
||||
o.OperationType == ReplicationOperationType.Remove && o.MessageId == result.MessageId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-016: an operator retrying a parked message must replicate a
|
||||
/// Requeue so the standby's copy moves back to Pending (otherwise it stays
|
||||
/// Parked on the standby and the operator's retry is lost across a failover).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RetryingAParkedMessage_ReplicatesARequeueOperation()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync(); // -> parked
|
||||
await WaitForReplicationAsync(2);
|
||||
|
||||
var retried = await _service.RetryParkedMessageAsync(result.MessageId);
|
||||
Assert.True(retried);
|
||||
|
||||
var ops = await WaitForReplicationAsync(3);
|
||||
var requeue = ops.SingleOrDefault(o =>
|
||||
o.OperationType == ReplicationOperationType.Requeue && o.MessageId == result.MessageId);
|
||||
Assert.NotNull(requeue);
|
||||
Assert.NotNull(requeue!.Message);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, requeue.Message!.Status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-016: the standby applies a Requeue by moving its row back to
|
||||
/// Pending with retry_count = 0, mirroring the active node's local state.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ApplyReplicatedOperation_Requeue_MovesStandbyRowBackToPending()
|
||||
{
|
||||
var replication = new ReplicationService(
|
||||
new StoreAndForwardOptions { ReplicationEnabled = true },
|
||||
NullLogger<ReplicationService>.Instance);
|
||||
|
||||
var parked = new StoreAndForwardMessage
|
||||
{
|
||||
Id = "requeue1",
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api",
|
||||
PayloadJson = "{}",
|
||||
RetryCount = 5,
|
||||
MaxRetries = 1,
|
||||
RetryIntervalMs = 0,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
};
|
||||
await _storage.EnqueueAsync(parked);
|
||||
|
||||
var requeued = new StoreAndForwardMessage
|
||||
{
|
||||
Id = parked.Id,
|
||||
Category = parked.Category,
|
||||
Target = parked.Target,
|
||||
PayloadJson = parked.PayloadJson,
|
||||
RetryCount = 0,
|
||||
MaxRetries = parked.MaxRetries,
|
||||
RetryIntervalMs = parked.RetryIntervalMs,
|
||||
CreatedAt = parked.CreatedAt,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
};
|
||||
await replication.ApplyReplicatedOperationAsync(
|
||||
new ReplicationOperation(ReplicationOperationType.Requeue, parked.Id, requeued),
|
||||
_storage);
|
||||
|
||||
var row = await _storage.GetMessageByIdAsync(parked.Id);
|
||||
Assert.NotNull(row);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, row!.Status);
|
||||
Assert.Equal(0, row.RetryCount);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,585 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-10/12/13/14: Tests for the StoreAndForwardService retry engine and management.
|
||||
/// </summary>
|
||||
public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
private readonly StoreAndForwardOptions _options;
|
||||
|
||||
public StoreAndForwardServiceTests()
|
||||
{
|
||||
var dbName = $"SvcTests_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
_options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 3,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10)
|
||||
};
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, _options, NullLogger<StoreAndForwardService>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
public void Dispose() => _keepAlive.Dispose();
|
||||
|
||||
// ── WP-10: Immediate delivery ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_ImmediateDeliverySuccess_ReturnsAcceptedNotBuffered()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api.example.com",
|
||||
"""{"method":"Test"}""", "Pump1");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.False(result.WasBuffered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_PermanentFailure_ReturnsNotAccepted()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(false));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api.example.com",
|
||||
"""{"method":"Test"}""");
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.False(result.WasBuffered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_TransientFailure_BuffersForRetry()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("Connection refused"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api.example.com",
|
||||
"""{"method":"Test"}""", "Pump1");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
// StoreAndForward-003: RetryCount counts sweep retries only; the immediate
|
||||
// attempt is attempt 0, so a freshly buffered message has RetryCount 0.
|
||||
Assert.Equal(0, msg.RetryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_NoHandler_BuffersForLater()
|
||||
{
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.Notification, "alerts@company.com",
|
||||
"""{"subject":"Alert"}""");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.True(result.WasBuffered);
|
||||
}
|
||||
|
||||
// ── WP-10: Retry engine ──
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_SuccessfulRetry_RemovesMessage()
|
||||
{
|
||||
int callCount = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ =>
|
||||
{
|
||||
callCount++;
|
||||
if (callCount == 1) throw new HttpRequestException("fail");
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_MaxRetriesReached_ParksMessage()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 2);
|
||||
|
||||
// StoreAndForward-003: MaxRetries bounds sweep retries (not the immediate
|
||||
// attempt), so a message with MaxRetries=2 needs two retry sweeps to park.
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
var afterFirst = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, afterFirst!.Status);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
}
|
||||
|
||||
// ── StoreAndForward-003: retry-count accounting ──
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_MaxRetriesOne_PerformsExactlyOneRetryBeforeParking()
|
||||
{
|
||||
// The immediate attempt is attempt 0; MaxRetries=1 must allow exactly one
|
||||
// retry sweep before parking. The pre-fix off-by-one parked with zero retries.
|
||||
var attempts = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => { Interlocked.Increment(ref attempts); throw new HttpRequestException("always fails"); });
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 1);
|
||||
|
||||
// After the immediate failed attempt the message is buffered, not parked.
|
||||
var buffered = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, buffered!.Status);
|
||||
Assert.Equal(1, attempts); // only the immediate attempt so far
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
Assert.Equal(2, attempts); // immediate attempt + exactly one retry
|
||||
Assert.Equal(1, msg.RetryCount); // one sweep retry recorded
|
||||
}
|
||||
|
||||
// ── StoreAndForward-005: sweep-vs-management race hardening ──
|
||||
|
||||
[Fact]
|
||||
public async Task RetryMessageAsync_StatusChangedDuringDelivery_SweepParkWriteIsSkipped()
|
||||
{
|
||||
// StoreAndForward-005: the retry sweep's state-changing writes must be
|
||||
// conditional on the status it observed, so a concurrent operator action that
|
||||
// moved the row out of Pending (e.g. between the sweep's snapshot load and its
|
||||
// park write) is not silently overwritten by the sweep's stale view.
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
attemptImmediateDelivery: false, maxRetries: 1);
|
||||
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
async msg =>
|
||||
{
|
||||
// Simulate an operator action winning the race: the row leaves Pending
|
||||
// (here: parked) while the sweep is still mid-delivery. The sweep would
|
||||
// otherwise unconditionally re-write this row from its stale snapshot.
|
||||
var parkedOutFromUnderTheSweep = new StoreAndForwardMessage
|
||||
{
|
||||
Id = msg.Id, Category = msg.Category, Target = msg.Target,
|
||||
PayloadJson = msg.PayloadJson, RetryCount = 7,
|
||||
MaxRetries = msg.MaxRetries, RetryIntervalMs = msg.RetryIntervalMs,
|
||||
CreatedAt = msg.CreatedAt, LastAttemptAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
LastError = "operator/other writer"
|
||||
};
|
||||
await _storage.UpdateMessageAsync(parkedOutFromUnderTheSweep);
|
||||
throw new HttpRequestException("transient — sweep will try to park");
|
||||
});
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
// The sweep observed Pending; the row is now Parked with the other writer's
|
||||
// RetryCount (7), not the sweep's (1). The sweep's conditional write was skipped.
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
Assert.Equal(7, msg.RetryCount);
|
||||
Assert.Equal("operator/other writer", msg.LastError);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_PermanentFailureOnRetry_ParksMessage()
|
||||
{
|
||||
int callCount = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ =>
|
||||
{
|
||||
callCount++;
|
||||
if (callCount == 1) throw new HttpRequestException("transient");
|
||||
return Task.FromResult(false);
|
||||
});
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
}
|
||||
|
||||
// ── WP-12: Parked message management ──
|
||||
|
||||
[Fact]
|
||||
public async Task RetryParkedMessageAsync_MovesBackToQueue()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 1);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
|
||||
var retried = await _service.RetryParkedMessageAsync(result.MessageId);
|
||||
Assert.True(retried);
|
||||
|
||||
msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
Assert.Equal(0, msg.RetryCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-017: the Retry activity-log entry must carry the parked
|
||||
/// message's true category, not a hard-coded ExternalSystem.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RetryParkedMessageAsync_ActivityUsesMessageRealCategory()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.Notification, "ops-list", """{}""",
|
||||
maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync(); // -> parked
|
||||
|
||||
var categories = new List<StoreAndForwardCategory>();
|
||||
_service.OnActivity += (action, category, _) =>
|
||||
{
|
||||
if (action == "Retry") categories.Add(category);
|
||||
};
|
||||
|
||||
var retried = await _service.RetryParkedMessageAsync(result.MessageId);
|
||||
Assert.True(retried);
|
||||
|
||||
Assert.Equal(new[] { StoreAndForwardCategory.Notification }, categories);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-017: the Discard activity-log entry must carry the parked
|
||||
/// message's true category, not a hard-coded ExternalSystem.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DiscardParkedMessageAsync_ActivityUsesMessageRealCategory()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.CachedDbWrite,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.CachedDbWrite, "site-db", """{}""",
|
||||
maxRetries: 1);
|
||||
await _service.RetryPendingMessagesAsync(); // -> parked
|
||||
|
||||
var categories = new List<StoreAndForwardCategory>();
|
||||
_service.OnActivity += (action, category, _) =>
|
||||
{
|
||||
if (action == "Discard") categories.Add(category);
|
||||
};
|
||||
|
||||
var discarded = await _service.DiscardParkedMessageAsync(result.MessageId);
|
||||
Assert.True(discarded);
|
||||
|
||||
Assert.Equal(new[] { StoreAndForwardCategory.CachedDbWrite }, categories);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscardParkedMessageAsync_PermanentlyRemoves()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 1);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var discarded = await _service.DiscardParkedMessageAsync(result.MessageId);
|
||||
Assert.True(discarded);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetParkedMessagesAsync_ReturnsPaginatedResults()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, $"api{i}", """{}""",
|
||||
maxRetries: 1);
|
||||
}
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var (messages, total) = await _service.GetParkedMessagesAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, 1, 2);
|
||||
|
||||
Assert.Equal(2, messages.Count);
|
||||
Assert.True(total >= 3);
|
||||
}
|
||||
|
||||
// ── WP-13: Messages survive instance deletion ──
|
||||
|
||||
[Fact]
|
||||
public async Task MessagesForInstance_SurviveAfterDeletion()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", "Pump1");
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api2", """{}""", "Pump1");
|
||||
|
||||
var count = await _service.GetMessageCountForInstanceAsync("Pump1");
|
||||
Assert.Equal(2, count);
|
||||
}
|
||||
|
||||
// ── WP-14: Health metrics ──
|
||||
|
||||
[Fact]
|
||||
public async Task GetBufferDepthAsync_ReturnsCorrectDepth()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api1", """{}""");
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api2", """{}""");
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.Notification, "email", """{}""");
|
||||
|
||||
var depth = await _service.GetBufferDepthAsync();
|
||||
Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.ExternalSystem) >= 2);
|
||||
Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.Notification) >= 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OnActivity_RaisedOnEnqueue()
|
||||
{
|
||||
var activities = new List<string>();
|
||||
_service.OnActivity += (action, _, _) => activities.Add(action);
|
||||
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
Assert.Contains("Delivered", activities);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OnActivity_RaisedOnBuffer()
|
||||
{
|
||||
var activities = new List<string>();
|
||||
_service.OnActivity += (action, _, _) => activities.Add(action);
|
||||
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
Assert.Contains("Queued", activities);
|
||||
}
|
||||
|
||||
// ── StoreAndForward-009: faulting activity subscriber must not corrupt delivery ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_ImmediateDeliverySuccess_FaultingActivitySubscriber_StillReportsDelivered()
|
||||
{
|
||||
// StoreAndForward-009: a throwing OnActivity subscriber (e.g. the site event
|
||||
// log) must not be misclassified as a transient delivery failure. Pre-fix the
|
||||
// subscriber's exception escaped RaiseActivity, was caught by EnqueueAsync's
|
||||
// transient-failure handler, and a successfully delivered message was buffered.
|
||||
_service.OnActivity += (_, _, _) => throw new InvalidOperationException("logging blew up");
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.False(result.WasBuffered); // delivered, NOT buffered
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Null(msg); // nothing left in the buffer
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryMessageAsync_FaultingActivitySubscriber_DoesNotIncrementRetryCount()
|
||||
{
|
||||
// StoreAndForward-009: a throwing subscriber raised after a successful retry
|
||||
// delivery must not be caught by the retry-failure handler and counted as a
|
||||
// transient failure.
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
attemptImmediateDelivery: false, maxRetries: 5);
|
||||
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
_service.OnActivity += (_, _, _) => throw new InvalidOperationException("logging blew up");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
// The retry succeeded; the message must be gone, not re-buffered with a bumped count.
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
// ── WP-10: Per-source-entity retry settings ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_CustomRetrySettings_Respected()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 100,
|
||||
retryInterval: TimeSpan.FromSeconds(60));
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(100, msg!.MaxRetries);
|
||||
Assert.Equal(60000, msg.RetryIntervalMs);
|
||||
}
|
||||
|
||||
// ── attemptImmediateDelivery: false — caller already attempted delivery ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_AttemptImmediateDeliveryFalse_BuffersWithoutInvokingHandler()
|
||||
{
|
||||
// A caller that has already made its own delivery attempt passes
|
||||
// attemptImmediateDelivery: false so the request is not dispatched twice.
|
||||
var handlerCalls = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => { Interlocked.Increment(ref handlerCalls); return Task.FromResult(true); });
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
attemptImmediateDelivery: false);
|
||||
|
||||
Assert.Equal(0, handlerCalls); // handler NOT invoked at enqueue time
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
// StoreAndForward-003: the caller's own attempt is attempt 0; RetryCount
|
||||
// counts only sweep retries, so a freshly buffered message has RetryCount 0.
|
||||
Assert.Equal(0, msg.RetryCount);
|
||||
}
|
||||
|
||||
// ─── StoreAndForward-024: StopAsync waits for the in-flight sweep ───
|
||||
|
||||
/// <summary>
|
||||
/// StoreAndForward-024: <see cref="StoreAndForwardService.StopAsync"/> must
|
||||
/// not return until any in-flight retry sweep has completed (or the bounded
|
||||
/// shutdown timeout fires). Pre-fix it disposed the timer and returned
|
||||
/// immediately, leaving a mid-flight sweep touching disposed dependencies.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task StopAsync_AwaitsInFlightRetrySweep_BeforeReturning()
|
||||
{
|
||||
// Build a service whose timer fires almost immediately, with a handler
|
||||
// that pauses in the middle of delivery so we can observe StopAsync's
|
||||
// wait behaviour.
|
||||
var dbName = $"StopWait_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
using var keepAlive = new SqliteConnection(connStr);
|
||||
keepAlive.Open();
|
||||
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 3,
|
||||
// Fire almost immediately so the sweep is in-flight by the time we call StopAsync.
|
||||
RetryTimerInterval = TimeSpan.FromMilliseconds(20),
|
||||
};
|
||||
var service = new StoreAndForwardService(
|
||||
storage, options, NullLogger<StoreAndForwardService>.Instance);
|
||||
|
||||
// Pre-seed a buffered message so the sweep has work to do, and a
|
||||
// handler that blocks until we release it.
|
||||
var handlerEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var releaseHandler = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var handlerCompleted = false;
|
||||
service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async _ =>
|
||||
{
|
||||
handlerEntered.TrySetResult();
|
||||
await releaseHandler.Task;
|
||||
handlerCompleted = true;
|
||||
return true;
|
||||
});
|
||||
|
||||
var seed = await service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
attemptImmediateDelivery: false);
|
||||
Assert.True(seed.WasBuffered);
|
||||
|
||||
await service.StartAsync();
|
||||
// Wait until the timer-driven sweep has called into the handler.
|
||||
var entered = await Task.WhenAny(handlerEntered.Task, Task.Delay(TimeSpan.FromSeconds(2)));
|
||||
Assert.Same(handlerEntered.Task, entered);
|
||||
Assert.False(handlerCompleted, "Handler should still be paused inside the sweep.");
|
||||
|
||||
// Kick StopAsync — it must NOT return until the sweep finishes. Run the
|
||||
// release on a background task so we can prove StopAsync is awaiting.
|
||||
var stopTask = service.StopAsync();
|
||||
Assert.False(stopTask.IsCompleted,
|
||||
"StopAsync returned before the in-flight sweep was given a chance to finish.");
|
||||
|
||||
// Release the handler — StopAsync should now complete shortly.
|
||||
releaseHandler.SetResult();
|
||||
await stopTask.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
Assert.True(handlerCompleted,
|
||||
"Sweep handler must have finished before StopAsync returned.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,632 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-9: Tests for SQLite persistence layer.
|
||||
/// Uses in-memory SQLite with a kept-alive connection for test isolation.
|
||||
/// </summary>
|
||||
public class StoreAndForwardStorageTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly string _dbName;
|
||||
|
||||
public StoreAndForwardStorageTests()
|
||||
{
|
||||
_dbName = $"StorageTests_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={_dbName};Mode=Memory;Cache=Shared";
|
||||
// Keep one connection alive so the in-memory DB persists
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_keepAlive.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_StoresMessage()
|
||||
{
|
||||
var message = CreateMessage("msg1", StoreAndForwardCategory.ExternalSystem);
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("msg1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal("msg1", retrieved!.Id);
|
||||
Assert.Equal(StoreAndForwardCategory.ExternalSystem, retrieved.Category);
|
||||
Assert.Equal("target1", retrieved.Target);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_AllCategories()
|
||||
{
|
||||
await _storage.EnqueueAsync(CreateMessage("es1", StoreAndForwardCategory.ExternalSystem));
|
||||
await _storage.EnqueueAsync(CreateMessage("n1", StoreAndForwardCategory.Notification));
|
||||
await _storage.EnqueueAsync(CreateMessage("db1", StoreAndForwardCategory.CachedDbWrite));
|
||||
|
||||
var es = await _storage.GetMessageByIdAsync("es1");
|
||||
var n = await _storage.GetMessageByIdAsync("n1");
|
||||
var db = await _storage.GetMessageByIdAsync("db1");
|
||||
|
||||
Assert.Equal(StoreAndForwardCategory.ExternalSystem, es!.Category);
|
||||
Assert.Equal(StoreAndForwardCategory.Notification, n!.Category);
|
||||
Assert.Equal(StoreAndForwardCategory.CachedDbWrite, db!.Category);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveMessageAsync_RemovesSuccessfully()
|
||||
{
|
||||
await _storage.EnqueueAsync(CreateMessage("rm1", StoreAndForwardCategory.ExternalSystem));
|
||||
await _storage.RemoveMessageAsync("rm1");
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("rm1");
|
||||
Assert.Null(retrieved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateMessageAsync_UpdatesFields()
|
||||
{
|
||||
var message = CreateMessage("upd1", StoreAndForwardCategory.ExternalSystem);
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
message.RetryCount = 5;
|
||||
message.LastAttemptAt = DateTimeOffset.UtcNow;
|
||||
message.Status = StoreAndForwardMessageStatus.Parked;
|
||||
message.LastError = "Connection refused";
|
||||
await _storage.UpdateMessageAsync(message);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("upd1");
|
||||
Assert.Equal(5, retrieved!.RetryCount);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, retrieved.Status);
|
||||
Assert.Equal("Connection refused", retrieved.LastError);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetMessagesForRetryAsync_ReturnsOnlyPendingMessages()
|
||||
{
|
||||
var pending = CreateMessage("pend1", StoreAndForwardCategory.ExternalSystem);
|
||||
pending.Status = StoreAndForwardMessageStatus.Pending;
|
||||
await _storage.EnqueueAsync(pending);
|
||||
|
||||
var parked = CreateMessage("park1", StoreAndForwardCategory.ExternalSystem);
|
||||
parked.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.EnqueueAsync(parked);
|
||||
await _storage.UpdateMessageAsync(parked);
|
||||
|
||||
var forRetry = await _storage.GetMessagesForRetryAsync();
|
||||
Assert.All(forRetry, m => Assert.Equal(StoreAndForwardMessageStatus.Pending, m.Status));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetMessagesForRetryAsync_NonZeroInterval_ExcludesNotYetDueIncludesDue()
|
||||
{
|
||||
// StoreAndForward-013: exercise the julianday elapsed-time comparison with a
|
||||
// non-zero retry interval. A message attempted just now must NOT be due; one
|
||||
// attempted long ago must be due.
|
||||
var notDue = CreateMessage("notdue", StoreAndForwardCategory.ExternalSystem);
|
||||
notDue.RetryIntervalMs = (long)TimeSpan.FromHours(1).TotalMilliseconds;
|
||||
notDue.LastAttemptAt = DateTimeOffset.UtcNow;
|
||||
await _storage.EnqueueAsync(notDue);
|
||||
|
||||
var due = CreateMessage("due", StoreAndForwardCategory.ExternalSystem);
|
||||
due.RetryIntervalMs = (long)TimeSpan.FromMinutes(5).TotalMilliseconds;
|
||||
due.LastAttemptAt = DateTimeOffset.UtcNow.AddHours(-2);
|
||||
await _storage.EnqueueAsync(due);
|
||||
|
||||
var neverAttempted = CreateMessage("never", StoreAndForwardCategory.ExternalSystem);
|
||||
neverAttempted.RetryIntervalMs = (long)TimeSpan.FromHours(1).TotalMilliseconds;
|
||||
neverAttempted.LastAttemptAt = null;
|
||||
await _storage.EnqueueAsync(neverAttempted);
|
||||
|
||||
var forRetry = await _storage.GetMessagesForRetryAsync();
|
||||
var ids = forRetry.Select(m => m.Id).ToHashSet();
|
||||
|
||||
Assert.DoesNotContain("notdue", ids);
|
||||
Assert.Contains("due", ids);
|
||||
Assert.Contains("never", ids);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetParkedMessagesAsync_ReturnsParkedOnly()
|
||||
{
|
||||
var msg = CreateMessage("prk1", StoreAndForwardCategory.Notification);
|
||||
msg.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.EnqueueAsync(msg);
|
||||
await _storage.UpdateMessageAsync(msg);
|
||||
|
||||
var (messages, total) = await _storage.GetParkedMessagesAsync();
|
||||
Assert.True(total > 0);
|
||||
Assert.All(messages, m => Assert.Equal(StoreAndForwardMessageStatus.Parked, m.Status));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryParkedMessageAsync_MovesToPending()
|
||||
{
|
||||
var msg = CreateMessage("retry1", StoreAndForwardCategory.ExternalSystem);
|
||||
msg.Status = StoreAndForwardMessageStatus.Parked;
|
||||
msg.RetryCount = 10;
|
||||
await _storage.EnqueueAsync(msg);
|
||||
await _storage.UpdateMessageAsync(msg);
|
||||
|
||||
var success = await _storage.RetryParkedMessageAsync("retry1");
|
||||
Assert.True(success);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("retry1");
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, retrieved!.Status);
|
||||
Assert.Equal(0, retrieved.RetryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryParkedMessageAsync_ClearsLastAttemptAt_SoMessageIsImmediatelyDue()
|
||||
{
|
||||
// StoreAndForward-010: a re-queued parked message must be unambiguously due
|
||||
// for the next sweep regardless of its (stale) last_attempt_at. Use a large
|
||||
// retry interval so a leftover timestamp would otherwise exclude the message.
|
||||
var msg = CreateMessage("requeue1", StoreAndForwardCategory.ExternalSystem);
|
||||
msg.RetryIntervalMs = (long)TimeSpan.FromHours(1).TotalMilliseconds;
|
||||
msg.LastAttemptAt = DateTimeOffset.UtcNow; // recent attempt
|
||||
msg.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.EnqueueAsync(msg);
|
||||
await _storage.UpdateMessageAsync(msg);
|
||||
|
||||
var requeued = await _storage.RetryParkedMessageAsync("requeue1");
|
||||
Assert.True(requeued);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("requeue1");
|
||||
Assert.Null(retrieved!.LastAttemptAt);
|
||||
|
||||
// It must appear in the retry-due set even though the configured interval
|
||||
// (1 hour) has not elapsed since the original attempt.
|
||||
var due = await _storage.GetMessagesForRetryAsync();
|
||||
Assert.Contains(due, m => m.Id == "requeue1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscardParkedMessageAsync_RemovesMessage()
|
||||
{
|
||||
var msg = CreateMessage("disc1", StoreAndForwardCategory.ExternalSystem);
|
||||
msg.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.EnqueueAsync(msg);
|
||||
await _storage.UpdateMessageAsync(msg);
|
||||
|
||||
var success = await _storage.DiscardParkedMessageAsync("disc1");
|
||||
Assert.True(success);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("disc1");
|
||||
Assert.Null(retrieved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetBufferDepthByCategoryAsync_ReturnsCorrectCounts()
|
||||
{
|
||||
await _storage.EnqueueAsync(CreateMessage("bd1", StoreAndForwardCategory.ExternalSystem));
|
||||
await _storage.EnqueueAsync(CreateMessage("bd2", StoreAndForwardCategory.ExternalSystem));
|
||||
await _storage.EnqueueAsync(CreateMessage("bd3", StoreAndForwardCategory.Notification));
|
||||
|
||||
var depth = await _storage.GetBufferDepthByCategoryAsync();
|
||||
Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.ExternalSystem) >= 2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetMessageCountByOriginInstanceAsync_ReturnsCount()
|
||||
{
|
||||
var msg1 = CreateMessage("oi1", StoreAndForwardCategory.ExternalSystem);
|
||||
msg1.OriginInstanceName = "Pump1";
|
||||
await _storage.EnqueueAsync(msg1);
|
||||
|
||||
var msg2 = CreateMessage("oi2", StoreAndForwardCategory.Notification);
|
||||
msg2.OriginInstanceName = "Pump1";
|
||||
await _storage.EnqueueAsync(msg2);
|
||||
|
||||
var count = await _storage.GetMessageCountByOriginInstanceAsync("Pump1");
|
||||
Assert.Equal(2, count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetParkedMessagesAsync_Pagination()
|
||||
{
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
var msg = CreateMessage($"page{i}", StoreAndForwardCategory.ExternalSystem);
|
||||
msg.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.EnqueueAsync(msg);
|
||||
await _storage.UpdateMessageAsync(msg);
|
||||
}
|
||||
|
||||
var (page1, total) = await _storage.GetParkedMessagesAsync(pageNumber: 1, pageSize: 2);
|
||||
Assert.Equal(2, page1.Count);
|
||||
Assert.True(total >= 5);
|
||||
|
||||
var (page2, _) = await _storage.GetParkedMessagesAsync(pageNumber: 2, pageSize: 2);
|
||||
Assert.Equal(2, page2.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetParkedMessagesAsync_TransactionedReads_CountMatchesFullResultSet()
|
||||
{
|
||||
// StoreAndForward-006: the COUNT(*) and paged SELECT now run inside one
|
||||
// transaction so they share a consistent snapshot. This functional check
|
||||
// guards the fix — it verifies the transaction wiring did not break paging:
|
||||
// the reported TotalCount and the rows assembled across all pages agree, and
|
||||
// a page wide enough to hold every parked row contains exactly TotalCount rows.
|
||||
for (int i = 0; i < 25; i++)
|
||||
{
|
||||
var m = CreateMessage($"txn-{i}", StoreAndForwardCategory.ExternalSystem);
|
||||
m.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.EnqueueAsync(m);
|
||||
await _storage.UpdateMessageAsync(m);
|
||||
}
|
||||
|
||||
var (wholePage, wholeTotal) = await _storage.GetParkedMessagesAsync(pageNumber: 1, pageSize: 1000);
|
||||
Assert.Equal(25, wholeTotal);
|
||||
Assert.Equal(wholeTotal, wholePage.Count);
|
||||
|
||||
var collected = new List<string>();
|
||||
int reportedTotal = -1;
|
||||
for (int page = 1; ; page++)
|
||||
{
|
||||
var (rows, total) = await _storage.GetParkedMessagesAsync(pageNumber: page, pageSize: 7);
|
||||
reportedTotal = total;
|
||||
collected.AddRange(rows.Select(r => r.Id));
|
||||
if (rows.Count < 7) break;
|
||||
}
|
||||
Assert.Equal(reportedTotal, collected.Count);
|
||||
Assert.Equal(25, collected.Distinct().Count());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetMessageCountByStatusAsync_ReturnsAccurateCount()
|
||||
{
|
||||
var msg = CreateMessage("cnt1", StoreAndForwardCategory.ExternalSystem);
|
||||
await _storage.EnqueueAsync(msg);
|
||||
|
||||
var count = await _storage.GetMessageCountByStatusAsync(StoreAndForwardMessageStatus.Pending);
|
||||
Assert.True(count >= 1);
|
||||
}
|
||||
|
||||
// ── Audit Log #23 (ExecutionId Task 4): execution_id / source_script ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_RoundTripsExecutionIdAndSourceScript()
|
||||
{
|
||||
// A cached call buffered on a transient failure carries the originating
|
||||
// script execution's ExecutionId + SourceScript; both must survive a
|
||||
// persist + read-back so the retry loop can stamp them on audit rows.
|
||||
var executionId = Guid.NewGuid();
|
||||
var message = CreateMessage("exec1", StoreAndForwardCategory.ExternalSystem);
|
||||
message.ExecutionId = executionId;
|
||||
message.SourceScript = "Plant.Pump42/OnTick";
|
||||
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("exec1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal(executionId, retrieved!.ExecutionId);
|
||||
Assert.Equal("Plant.Pump42/OnTick", retrieved.SourceScript);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_NullExecutionIdAndSourceScript_RoundTripAsNull()
|
||||
{
|
||||
// Non-cached-call enqueues (notifications) supply neither field — they
|
||||
// must round-trip as null rather than throwing or coercing.
|
||||
var message = CreateMessage("noexec1", StoreAndForwardCategory.Notification);
|
||||
Assert.Null(message.ExecutionId);
|
||||
Assert.Null(message.SourceScript);
|
||||
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("noexec1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Null(retrieved!.ExecutionId);
|
||||
Assert.Null(retrieved.SourceScript);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecutionIdAndSourceScript_SurviveRetrySweepRead()
|
||||
{
|
||||
// The retry sweep reads due rows via GetMessagesForRetryAsync; the new
|
||||
// fields must be present on that read path too (it is the path that
|
||||
// feeds the CachedCallAttemptContext).
|
||||
var executionId = Guid.NewGuid();
|
||||
var message = CreateMessage("sweep1", StoreAndForwardCategory.CachedDbWrite);
|
||||
message.ExecutionId = executionId;
|
||||
message.SourceScript = "Plant.Tank/OnAlarm";
|
||||
message.LastAttemptAt = null; // due immediately
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var due = await _storage.GetMessagesForRetryAsync();
|
||||
|
||||
var row = Assert.Single(due, m => m.Id == "sweep1");
|
||||
Assert.Equal(executionId, row.ExecutionId);
|
||||
Assert.Equal("Plant.Tank/OnAlarm", row.SourceScript);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LegacyRowWithoutNewColumns_ReadsBackAsNull()
|
||||
{
|
||||
// Back-compat: a row persisted by a build that pre-dates the
|
||||
// execution_id / source_script columns must still deserialize, with
|
||||
// ExecutionId / SourceScript reading back as null. Simulate the legacy
|
||||
// schema by dropping the table and recreating it without the columns,
|
||||
// inserting directly, then running InitializeAsync (which ALTER-adds
|
||||
// the columns) and reading the row back.
|
||||
await using (var setup = new SqliteConnection($"Data Source={_dbName};Mode=Memory;Cache=Shared"))
|
||||
{
|
||||
await setup.OpenAsync();
|
||||
await using var drop = setup.CreateCommand();
|
||||
drop.CommandText = @"
|
||||
DROP TABLE IF EXISTS sf_messages;
|
||||
CREATE TABLE sf_messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
category INTEGER NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_retries INTEGER NOT NULL DEFAULT 50,
|
||||
retry_interval_ms INTEGER NOT NULL DEFAULT 30000,
|
||||
created_at TEXT NOT NULL,
|
||||
last_attempt_at TEXT,
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
origin_instance TEXT
|
||||
);
|
||||
INSERT INTO sf_messages (id, category, target, payload_json, created_at, status)
|
||||
VALUES ('legacy1', 0, 'ERP', '{}', '2026-01-01T00:00:00.0000000+00:00', 0);";
|
||||
await drop.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
// InitializeAsync must additively ALTER-in the new columns without
|
||||
// disturbing the pre-existing legacy row.
|
||||
await _storage.InitializeAsync();
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("legacy1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal("legacy1", retrieved!.Id);
|
||||
Assert.Null(retrieved.ExecutionId);
|
||||
Assert.Null(retrieved.SourceScript);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MalformedExecutionId_ReadsBackAsNull_DoesNotAbortRetrySweep()
|
||||
{
|
||||
// Defensive read path: a corrupt (non-null, non-GUID) execution_id must
|
||||
// be treated as "no execution id" rather than throwing FormatException
|
||||
// — a single bad row must not abort the whole GetMessagesForRetryAsync
|
||||
// sweep, which reads many rows. Persist two due rows, then corrupt the
|
||||
// execution_id of one directly in the DB.
|
||||
var goodId = Guid.NewGuid();
|
||||
var good = CreateMessage("good1", StoreAndForwardCategory.ExternalSystem);
|
||||
good.ExecutionId = goodId;
|
||||
good.LastAttemptAt = null; // due immediately
|
||||
await _storage.EnqueueAsync(good);
|
||||
|
||||
var bad = CreateMessage("bad1", StoreAndForwardCategory.ExternalSystem);
|
||||
bad.ExecutionId = Guid.NewGuid();
|
||||
bad.LastAttemptAt = null; // due immediately
|
||||
await _storage.EnqueueAsync(bad);
|
||||
|
||||
await using (var conn = new SqliteConnection($"Data Source={_dbName};Mode=Memory;Cache=Shared"))
|
||||
{
|
||||
await conn.OpenAsync();
|
||||
await using var corrupt = conn.CreateCommand();
|
||||
corrupt.CommandText =
|
||||
"UPDATE sf_messages SET execution_id = 'not-a-guid' WHERE id = 'bad1';";
|
||||
await corrupt.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
// The sweep must not throw; the corrupt row reads back with a null
|
||||
// ExecutionId, the well-formed row keeps its value.
|
||||
var due = await _storage.GetMessagesForRetryAsync();
|
||||
Assert.Null(Assert.Single(due, m => m.Id == "bad1").ExecutionId);
|
||||
Assert.Equal(goodId, Assert.Single(due, m => m.Id == "good1").ExecutionId);
|
||||
|
||||
// The single-row read path is equally defensive.
|
||||
var retrieved = await _storage.GetMessageByIdAsync("bad1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Null(retrieved!.ExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InitializeAsync_IsIdempotent_WhenColumnsAlreadyExist()
|
||||
{
|
||||
// The additive ALTER must not fail on a second InitializeAsync call
|
||||
// (SQLite has no ADD COLUMN IF NOT EXISTS — the probe must skip it).
|
||||
await _storage.InitializeAsync();
|
||||
await _storage.InitializeAsync();
|
||||
|
||||
var message = CreateMessage("idem1", StoreAndForwardCategory.ExternalSystem);
|
||||
message.ExecutionId = Guid.NewGuid();
|
||||
await _storage.EnqueueAsync(message);
|
||||
var retrieved = await _storage.GetMessageByIdAsync("idem1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal(message.ExecutionId, retrieved!.ExecutionId);
|
||||
}
|
||||
|
||||
// ── Audit Log #23 (ParentExecutionId Task 6): parent_execution_id ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_RoundTripsParentExecutionId()
|
||||
{
|
||||
// A cached call buffered from an inbound-API-routed script run carries
|
||||
// the spawning execution's ParentExecutionId; it must survive a persist
|
||||
// + read-back so the retry loop can stamp it on audit rows.
|
||||
var parentExecutionId = Guid.NewGuid();
|
||||
var message = CreateMessage("parent1", StoreAndForwardCategory.ExternalSystem);
|
||||
message.ParentExecutionId = parentExecutionId;
|
||||
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("parent1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal(parentExecutionId, retrieved!.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_NullParentExecutionId_RoundTripsAsNull()
|
||||
{
|
||||
// A non-routed run supplies no ParentExecutionId — it must round-trip
|
||||
// as null rather than throwing or coercing.
|
||||
var message = CreateMessage("noparent1", StoreAndForwardCategory.ExternalSystem);
|
||||
Assert.Null(message.ParentExecutionId);
|
||||
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("noparent1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Null(retrieved!.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ParentExecutionId_SurvivesRetrySweepRead()
|
||||
{
|
||||
// The retry sweep reads due rows via GetMessagesForRetryAsync; the new
|
||||
// parent_execution_id field must be present on that read path too — it
|
||||
// is the path that feeds the CachedCallAttemptContext.
|
||||
var parentExecutionId = Guid.NewGuid();
|
||||
var message = CreateMessage("psweep1", StoreAndForwardCategory.CachedDbWrite);
|
||||
message.ParentExecutionId = parentExecutionId;
|
||||
message.LastAttemptAt = null; // due immediately
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
var due = await _storage.GetMessagesForRetryAsync();
|
||||
|
||||
var row = Assert.Single(due, m => m.Id == "psweep1");
|
||||
Assert.Equal(parentExecutionId, row.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LegacyRowWithoutParentExecutionIdColumn_ReadsBackAsNull()
|
||||
{
|
||||
// Back-compat: a row persisted by a build that pre-dates the
|
||||
// parent_execution_id column must still deserialize, with
|
||||
// ParentExecutionId reading back as null. Simulate the pre-Task-6
|
||||
// schema (which already has execution_id / source_script from the
|
||||
// ExecutionId rollout) by recreating the table without
|
||||
// parent_execution_id, inserting directly, then running InitializeAsync
|
||||
// which ALTER-adds the column.
|
||||
await using (var setup = new SqliteConnection($"Data Source={_dbName};Mode=Memory;Cache=Shared"))
|
||||
{
|
||||
await setup.OpenAsync();
|
||||
await using var drop = setup.CreateCommand();
|
||||
drop.CommandText = @"
|
||||
DROP TABLE IF EXISTS sf_messages;
|
||||
CREATE TABLE sf_messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
category INTEGER NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_retries INTEGER NOT NULL DEFAULT 50,
|
||||
retry_interval_ms INTEGER NOT NULL DEFAULT 30000,
|
||||
created_at TEXT NOT NULL,
|
||||
last_attempt_at TEXT,
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
origin_instance TEXT,
|
||||
execution_id TEXT,
|
||||
source_script TEXT
|
||||
);
|
||||
INSERT INTO sf_messages (id, category, target, payload_json, created_at, status)
|
||||
VALUES ('plegacy1', 0, 'ERP', '{}', '2026-01-01T00:00:00.0000000+00:00', 0);";
|
||||
await drop.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
// InitializeAsync must additively ALTER-in parent_execution_id without
|
||||
// disturbing the pre-existing legacy row.
|
||||
await _storage.InitializeAsync();
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("plegacy1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal("plegacy1", retrieved!.Id);
|
||||
Assert.Null(retrieved.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MalformedParentExecutionId_ReadsBackAsNull_DoesNotAbortRetrySweep()
|
||||
{
|
||||
// Defensive read path: a corrupt (non-null, non-GUID) parent_execution_id
|
||||
// must be treated as "no parent execution id" rather than throwing
|
||||
// FormatException — a single bad row must not abort the whole
|
||||
// GetMessagesForRetryAsync sweep.
|
||||
var goodParent = Guid.NewGuid();
|
||||
var good = CreateMessage("pgood1", StoreAndForwardCategory.ExternalSystem);
|
||||
good.ParentExecutionId = goodParent;
|
||||
good.LastAttemptAt = null; // due immediately
|
||||
await _storage.EnqueueAsync(good);
|
||||
|
||||
var bad = CreateMessage("pbad1", StoreAndForwardCategory.ExternalSystem);
|
||||
bad.ParentExecutionId = Guid.NewGuid();
|
||||
bad.LastAttemptAt = null; // due immediately
|
||||
await _storage.EnqueueAsync(bad);
|
||||
|
||||
await using (var conn = new SqliteConnection($"Data Source={_dbName};Mode=Memory;Cache=Shared"))
|
||||
{
|
||||
await conn.OpenAsync();
|
||||
await using var corrupt = conn.CreateCommand();
|
||||
corrupt.CommandText =
|
||||
"UPDATE sf_messages SET parent_execution_id = 'not-a-guid' WHERE id = 'pbad1';";
|
||||
await corrupt.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
var due = await _storage.GetMessagesForRetryAsync();
|
||||
Assert.Null(Assert.Single(due, m => m.Id == "pbad1").ParentExecutionId);
|
||||
Assert.Equal(goodParent, Assert.Single(due, m => m.Id == "pgood1").ParentExecutionId);
|
||||
|
||||
var retrieved = await _storage.GetMessageByIdAsync("pbad1");
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Null(retrieved!.ParentExecutionId);
|
||||
}
|
||||
|
||||
private static StoreAndForwardMessage CreateMessage(string id, StoreAndForwardCategory category)
|
||||
{
|
||||
return new StoreAndForwardMessage
|
||||
{
|
||||
Id = id,
|
||||
Category = category,
|
||||
Target = "target1",
|
||||
PayloadJson = """{"method":"Test","args":{}}""",
|
||||
RetryCount = 0,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending
|
||||
};
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InitializeAsync_FileInMissingDirectory_CreatesDirectory()
|
||||
{
|
||||
// SQLite creates the database file on demand but not its parent directory;
|
||||
// the storage must create the directory itself or OpenAsync fails with
|
||||
// "unable to open database file" (the cause of the SiteActorPathTests failures).
|
||||
var directory = Path.Combine(Path.GetTempPath(), "sf-storage-test-" + Guid.NewGuid().ToString("N"));
|
||||
var dbPath = Path.Combine(directory, "store-and-forward.db");
|
||||
Assert.False(Directory.Exists(directory));
|
||||
|
||||
try
|
||||
{
|
||||
var storage = new StoreAndForwardStorage(
|
||||
$"Data Source={dbPath}", NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
await storage.InitializeAsync();
|
||||
|
||||
Assert.True(Directory.Exists(directory));
|
||||
Assert.True(File.Exists(dbPath));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka.TestKit.Xunit2" />
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.StoreAndForward/ZB.MOM.WW.ScadaBridge.StoreAndForward.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
|
||||
"parallelizeTestCollections": false
|
||||
}
|
||||
Reference in New Issue
Block a user