5d075f1374
Six adversarial-review findings in the central SQL/ingest layer. F1 (AuditLogRepository.InsertChunkAsync) — the set-based ingest declared each string parameter at its COLUMN width (Actor/Target 256, Action 64, Outcome 16, Category 32, SourceNode 64), so SqlClient truncated an over-long value at bind time and committed the mutilated row — silent, in an append-only store, with no PayloadTruncated flag — while the per-row and reconciliation paths sent the same value in full and let the server reject it with 2628. Bind at the value's own length instead; explicit SqlDbType is kept (it fixes the VALUES constructor's derived column types and datetime2 precision). Design: reject everywhere, truncate nowhere — matching today's per-row behaviour. F2 (SiteCallAuditRepository.UpsertAsync) — the single-statement upsert ran the monotonic UPDATE first and INSERTed only if nothing matched. Two writers racing the first packet of one TrackedOperationId (the cached dual-write and the reconciliation pull carry DIFFERENT lifecycle states) both matched nothing, and the loser then skipped its INSERT or swallowed a 2627 — dropping its Status/RetryCount/HttpStatus/TerminalAtUtc. Legs swapped to `IF NOT EXISTS … INSERT; UPDATE <monotonic>` — still one round trip, and the loser's UPDATE now lands on the winner's row. The duplicate-key catch re-runs the monotonic UPDATE for the same reason. Moved to raw SQL with explicitly-typed parameters so the intricate rank predicate exists in exactly one place (an untyped DateTime would bind as `datetime` and round the freshness tiebreaker). F3 (docs/plans/sql/*.sql) — filtered-index DDL failed with error 1934 under the documented `docker exec … sqlcmd` path, which defaults QUOTED_IDENTIFIER OFF; once IX_Notifications_Delivered exists, QI-OFF DML on Notifications fails too. All four scripts now open with `SET QUOTED_IDENTIFIER ON; SET ANSI_NULLS ON; GO` (own batch, so it is in force when the next batch parses), and the migration convention in Component-ConfigurationDatabase.md documents `sqlcmd -I`. Verified live: the pre-fix script fails 1934 without -I, the fixed one applies. F4 (SiteCallAuditActor) — the off-mailbox reconciliation/purge passes reuse the injected repository, so tests drove one DbContext from the pass and a mailbox handler concurrently. Serialized at the CALL via a private SerializedRepository wrapper applied only by the test constructors, rather than running the pass on-mailbox: production keeps its PipeTo shape untouched, and the existing "a blocked drain does not stall ingest/query/KPI" regression tests stay meaningful (they would have been invalidated by suspending the mailbox). F5 (AuditLogIngestActor) — when the batch failed because the 20 s IngestBudget expired, the per-row fallback reused the same expired token: N instant failures, N counter bumps, zero accepted. The fallback now gets a fresh 5 s budget (inside the 30 s outer Ask), and a blown budget bumps the failure counter ONCE for the batch instead of once per row. F6 (NotificationOutboxRepository.UpdateAsync) — ExecuteUpdate's row count was discarded, so an operator Retry/Discard of a notification the retention purge had already deleted reported success (the pre-ExecuteUpdate code threw DbUpdateConcurrencyException). UpdateAsync now returns whether a row matched; the operator one-shots answer "notification not found" and emit no audit row for the action that did not happen, while the dispatcher logs a warning (its delivery already happened; nothing to retry). GetByIdAsync switched to AsNoTracking since the write is out-of-band. Tests: 5 new SQL-backed regressions (over-long Target rejected on both paths + boundary round-trip; concurrent first-write and already-created-by-another-writer upserts; vanished-row UpdateAsync), a token-identity pin on the ingest fallback, a repository-concurrency detector for the SiteCallAudit passes, and vanished-row operator-path tests. The F1/F2/F4 regressions were each confirmed failing against the pre-fix code. Suites: ConfigurationDatabase 369, AuditLog 378, SiteCallAudit 66, NotificationOutbox 152 — all green, solution builds with 0 warnings.
348 lines
14 KiB
C#
348 lines
14 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit.Xunit2;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using NSubstitute;
|
|
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
|
|
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
|
|
|
|
/// <summary>
|
|
/// M4 Bundle B (B2) — verifies the <see cref="NotificationOutboxActor"/>
|
|
/// dispatcher loop emits exactly ONE
|
|
/// <see cref="AuditChannel.Notification"/>/<see cref="AuditKind.NotifyDeliver"/>
|
|
/// audit row with <see cref="AuditStatus.Attempted"/> per attempt regardless of
|
|
/// the delivery outcome (success, transient, permanent). Terminal-state
|
|
/// emission is covered separately in
|
|
/// <see cref="NotificationOutboxActorTerminalEmissionTests"/>.
|
|
/// </summary>
|
|
public class NotificationOutboxActorAttemptEmissionTests : TestKit
|
|
{
|
|
private readonly INotificationOutboxRepository _outboxRepository =
|
|
OutboxRepositorySubstitute.Healthy();
|
|
|
|
private readonly INotificationRepository _notificationRepository =
|
|
Substitute.For<INotificationRepository>();
|
|
|
|
private readonly RecordingCentralAuditWriter _auditWriter = new();
|
|
|
|
/// <summary>
|
|
/// Recording writer so each test can assert on the events captured during
|
|
/// one dispatch tick without depending on a concrete implementation.
|
|
/// </summary>
|
|
private sealed class RecordingCentralAuditWriter : ICentralAuditWriter
|
|
{
|
|
// C3 (Task 2.5): store the decomposed row view so assertions keep
|
|
// reading the ScadaBridge domain fields (Channel/Kind/Status/…) as
|
|
// typed properties; the canonical record carries them in DetailsJson.
|
|
public List<AuditRowProjection.AuditRowValues> Events { get; } = new();
|
|
public Func<AuditEvent, Task>? OnWrite { get; set; }
|
|
|
|
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
|
{
|
|
lock (Events)
|
|
{
|
|
Events.Add(evt.AsRow());
|
|
}
|
|
|
|
return OnWrite?.Invoke(evt) ?? Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
private IServiceProvider BuildServiceProvider(IEnumerable<INotificationDeliveryAdapter> adapters)
|
|
{
|
|
var services = new ServiceCollection();
|
|
services.AddScoped(_ => _outboxRepository);
|
|
services.AddScoped(_ => _notificationRepository);
|
|
foreach (var adapter in adapters)
|
|
{
|
|
services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
|
|
}
|
|
|
|
return services.BuildServiceProvider();
|
|
}
|
|
|
|
private sealed class StubAdapter : INotificationDeliveryAdapter
|
|
{
|
|
private readonly Func<DeliveryOutcome> _outcome;
|
|
public int CallCount;
|
|
|
|
public StubAdapter(Func<DeliveryOutcome> outcome) { _outcome = outcome; }
|
|
|
|
public NotificationType Type => NotificationType.Email;
|
|
|
|
public Task<DeliveryOutcome> DeliverAsync(
|
|
Notification notification, CancellationToken cancellationToken = default)
|
|
{
|
|
Interlocked.Increment(ref CallCount);
|
|
return Task.FromResult(_outcome());
|
|
}
|
|
}
|
|
|
|
private IActorRef CreateActor(IEnumerable<INotificationDeliveryAdapter> adapters)
|
|
{
|
|
return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
|
BuildServiceProvider(adapters),
|
|
new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) },
|
|
(ICentralAuditWriter)_auditWriter,
|
|
NullLogger<NotificationOutboxActor>.Instance)));
|
|
}
|
|
|
|
private static Notification MakeNotification(
|
|
Guid? notificationId = null,
|
|
string sourceSite = "site-1",
|
|
int retryCount = 0,
|
|
Guid? originExecutionId = null,
|
|
Guid? originParentExecutionId = null)
|
|
{
|
|
return new Notification(
|
|
(notificationId ?? Guid.NewGuid()).ToString("D"),
|
|
NotificationType.Email,
|
|
"ops-team",
|
|
"Tank overflow",
|
|
"Tank 3 level critical",
|
|
sourceSite)
|
|
{
|
|
RetryCount = retryCount,
|
|
CreatedAt = DateTimeOffset.UtcNow,
|
|
SourceInstanceId = "instance-42",
|
|
SourceScript = "AlarmScript",
|
|
OriginExecutionId = originExecutionId,
|
|
OriginParentExecutionId = originParentExecutionId,
|
|
};
|
|
}
|
|
|
|
private void SetupSmtpRetryPolicy(int maxRetries, TimeSpan retryDelay)
|
|
{
|
|
var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
|
|
{
|
|
MaxRetries = maxRetries,
|
|
RetryDelay = retryDelay,
|
|
};
|
|
_notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[] { config });
|
|
}
|
|
|
|
private List<AuditRowProjection.AuditRowValues> EventsByStatus(AuditStatus status)
|
|
{
|
|
lock (_auditWriter.Events)
|
|
{
|
|
return _auditWriter.Events.Where(e => e.Status == status).ToList();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_Success_EmitsOneEvent_KindNotifyDeliver_StatusAttempted()
|
|
{
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var id = Guid.NewGuid();
|
|
var notification = MakeNotification(notificationId: id, sourceSite: "site-alpha");
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
var evt = attempted[0];
|
|
Assert.Equal(AuditChannel.Notification, evt.Channel);
|
|
Assert.Equal(AuditKind.NotifyDeliver, evt.Kind);
|
|
Assert.Equal(id, evt.CorrelationId);
|
|
Assert.Equal("ops-team", evt.Target);
|
|
Assert.Equal("site-alpha", evt.SourceSiteId);
|
|
Assert.Equal("instance-42", evt.SourceInstanceId);
|
|
Assert.Equal("AlarmScript", evt.SourceScript);
|
|
// Central dispatch: Actor is the system identity (no per-call user).
|
|
Assert.Equal("system", evt.Actor);
|
|
// Successful attempt: no error message.
|
|
Assert.Null(evt.ErrorMessage);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_CarriesOriginExecutionId_AsExecutionId()
|
|
{
|
|
// Audit Log #23: the Attempted NotifyDeliver row must echo the
|
|
// notification's OriginExecutionId so all rows for one run share an id.
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var executionId = Guid.NewGuid();
|
|
var notification = MakeNotification(originExecutionId: executionId);
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Equal(executionId, attempted[0].ExecutionId);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_NullOriginExecutionId_HasNullExecutionId()
|
|
{
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var notification = MakeNotification(originExecutionId: null);
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Null(attempted[0].ExecutionId);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_CarriesOriginParentExecutionId_AsParentExecutionId()
|
|
{
|
|
// Audit Log ParentExecutionId: the Attempted NotifyDeliver row must echo
|
|
// the notification's OriginParentExecutionId so the central dispatcher's
|
|
// rows carry the routed run's parent id.
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var parentExecutionId = Guid.NewGuid();
|
|
var notification = MakeNotification(originParentExecutionId: parentExecutionId);
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Equal(parentExecutionId, attempted[0].ParentExecutionId);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_NullOriginParentExecutionId_HasNullParentExecutionId()
|
|
{
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var notification = MakeNotification(originParentExecutionId: null);
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Null(attempted[0].ParentExecutionId);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_TransientFailure_EmitsEvent_StatusAttempted_ErrorMessageSet()
|
|
{
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var notification = MakeNotification(retryCount: 1);
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Equal(AuditKind.NotifyDeliver, attempted[0].Kind);
|
|
Assert.Equal("smtp timeout", attempted[0].ErrorMessage);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_PermanentFailure_EmitsEvent_StatusAttempted_ErrorMessageSet()
|
|
{
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var notification = MakeNotification();
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Permanent("invalid recipient address"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Equal(AuditKind.NotifyDeliver, attempted[0].Kind);
|
|
Assert.Equal("invalid recipient address", attempted[0].ErrorMessage);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void AuditWriter_Throws_DeliveryStateUpdate_StillSucceeds()
|
|
{
|
|
// Audit failure must NEVER abort the user-facing action: the delivery
|
|
// outcome must still be persisted via UpdateAsync.
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var notification = MakeNotification();
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
_auditWriter.OnWrite = _ => throw new InvalidOperationException("audit dead");
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
// Update of the notification row must still happen.
|
|
AwaitAssert(() =>
|
|
{
|
|
_outboxRepository.Received(1).UpdateAsync(
|
|
Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
|
|
Arg.Any<CancellationToken>());
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Attempt_RecordsOccurredAtUtc_AsUtc()
|
|
{
|
|
// The OccurredAtUtc on the emitted event must be UTC (all timestamps
|
|
// are UTC throughout the system).
|
|
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
|
|
var notification = MakeNotification();
|
|
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
|
.Returns(new[] { notification });
|
|
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
|
|
var actor = CreateActor([adapter]);
|
|
|
|
actor.Tell(InternalMessages.DispatchTick.Instance);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var attempted = EventsByStatus(AuditStatus.Attempted);
|
|
Assert.Single(attempted);
|
|
Assert.Equal(DateTimeKind.Utc, attempted[0].OccurredAtUtc.Kind);
|
|
});
|
|
}
|
|
}
|