dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'worktree-agent-adf34e265d2dcae96' into arch-review-remediation

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 23:47:00 -04:00
parent 95f091cff8 5d075f1374
commit 09e350ab0d
30 changed files with 1042 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogIngestActor.cs
+48 -6
View File
@@ -84,6 +84,22 @@ public class AuditLogIngestActor : ReceiveActor
/// </summary>
internal static readonly TimeSpan IngestSqlCommandTimeout = TimeSpan.FromSeconds(15);
/// <summary>
/// Budget for the per-row fallback loop, granted as a FRESH
/// <see cref="CancellationTokenSource"/> rather than reusing the batch's.
/// </summary>
/// <remarks>
/// The fallback used to run on the batch's own token. When the batch failed
/// BECAUSE that token expired, every fallback insert was handed an
/// already-cancelled token and failed instantly: N logged errors, N counter
/// bumps, zero rows accepted, and the site retried the whole packet anyway.
/// A fresh, deliberately SHORT budget (shorter than <see cref="IngestBudget"/>,
/// so the reply still beats the caller's Ask even when the batch consumed its
/// whole allowance) gives the poison-row isolation the fallback exists for a
/// real chance to land the good rows.
/// </remarks>
internal static readonly TimeSpan IngestFallbackBudget = TimeSpan.FromSeconds(5);
private readonly IServiceProvider? _serviceProvider;
private readonly IAuditLogRepository? _injectedRepository;
private readonly ILogger<AuditLogIngestActor> _logger;
@@ -265,15 +281,36 @@ public class AuditLogIngestActor : ReceiveActor
// back to the per-row path here, so a poison row still costs only
// itself. Re-running rows the failed batch may already have committed
// is safe: every insert is idempotent on EventId.
//
// The fallback gets its OWN token. Reusing the batch's meant that a
// batch which failed because the budget EXPIRED handed the loop an
// already-cancelled token, so every row failed instantly and the
// fallback did nothing but multiply the log and counter noise by the
// row count.
var budgetExpired = budget.IsCancellationRequested;
using var fallbackBudget = new CancellationTokenSource(IngestFallbackBudget);
_logger.LogWarning(ex,
"Set-based ingest of {Count} audit event(s) failed; falling back to per-row inserts so one bad row does not sink the batch.",
cmd.Events.Count);
"Set-based ingest of {Count} audit event(s) failed{BudgetNote}; falling back to per-row inserts (fresh {FallbackSeconds}s budget) so one bad row does not sink the batch.",
cmd.Events.Count,
budgetExpired ? " after exhausting the ingest budget" : string.Empty,
IngestFallbackBudget.TotalSeconds);
if (budgetExpired)
{
// A blown budget is ONE failure event for the batch, not one per
// row — bump the health counter once here and suppress the
// per-row bumps below so the dashboard sees a timeout as a
// timeout rather than as N independent write failures.
try { failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
}
for (var i = 0; i < projected.Count; i++)
{
try
{
await repository.InsertIfNotExistsAsync(projected[i], budget.Token).ConfigureAwait(false);
await repository.InsertIfNotExistsAsync(projected[i], fallbackBudget.Token).ConfigureAwait(false);
accepted.Add(cmd.Events[i].EventId);
}
catch (Exception rowEx)
@@ -281,9 +318,14 @@ public class AuditLogIngestActor : ReceiveActor
// Per-row catch — one bad row never sinks the whole batch.
// The row stays Pending at the site; the next drain retries.
// Bump the central health counter so a sustained insert-throw
// failure surfaces on the dashboard.
try { failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
// failure surfaces on the dashboard — unless the batch already
// counted itself as a single budget-exhaustion failure.
if (!budgetExpired)
{
try { failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
}
_logger.LogError(rowEx,
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
cmd.Events[i].EventId);
src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/INotificationOutboxRepository.cs
+16 -2
View File
@@ -63,11 +63,25 @@ public interface INotificationOutboxRepository
/// payload columns and the dispatcher writes on EVERY attempt. A future
/// caller that needs to change an immutable column must add its own method
/// rather than widening this one.
/// <para>
/// <b>The return value is the not-found signal.</b> A targeted UPDATE against
/// a row that no longer exists (the retention purge removed it between the
/// read and the write) affects zero rows and is otherwise indistinguishable
/// from success — which is how an operator Retry/Discard came to report
/// success against a vanished notification. Implementations MUST return
/// <see langword="false"/> when no row matched. Callers that speak to a human
/// (the operator one-shots) must surface that as "not found"; the dispatcher
/// treats it as a lost race and logs.
/// </para>
/// </remarks>
/// <param name="n">The notification whose delivery state should be persisted.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A task that completes when the notification has been persisted.</returns>
Task UpdateAsync(Notification n, CancellationToken cancellationToken = default);
/// <returns>
/// <see langword="true"/> when the row was found and its delivery state
/// persisted; <see langword="false"/> when no row with that
/// <c>NotificationId</c> exists any more.
/// </returns>
Task<bool> UpdateAsync(Notification n, CancellationToken cancellationToken = default);
/// <summary>
/// Returns a page of notifications matching <paramref name="filter"/>, ordered by
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs
+28 -6
View File
@@ -261,14 +261,28 @@ VALUES
var occurred = DateTime.SpecifyKind(evt.OccurredAtUtc.UtcDateTime, DateTimeKind.Utc);
object actor = string.IsNullOrEmpty(evt.Actor) ? DBNull.Value : evt.Actor;
// NO explicit Size on the string parameters — size 0 means "bind
// the value's own length" (see AddParameter). Declaring the
// COLUMN width here (Actor/Target 256, Action 64, Outcome 16,
// Category 32, SourceNode 64) made SqlClient TRUNCATE an
// over-long value client-side and commit the mutilated row, while
// the single-row path (ExecuteSqlInterpolated, no Size) and the
// reconciliation path sent the same value in full and let the
// server reject it with error 2628. An append-only audit store
// must never silently store a shortened row — there is no
// truncation flag on the record to say it happened — so the batch
// path now defers the length check to the server exactly like
// every other path: reject everywhere, truncate nowhere.
// The explicit SqlDbType stays (it is what fixes the VALUES
// constructor's derived column types, and DateTime2 precision).
AddParameter(cmd, "@e" + i, SqlDbType.UniqueIdentifier, size: 0, evt.EventId);
AddParameter(cmd, "@t" + i, SqlDbType.DateTime2, size: 0, occurred);
AddParameter(cmd, "@a" + i, SqlDbType.NVarChar, size: 256, actor);
AddParameter(cmd, "@n" + i, SqlDbType.VarChar, size: 64, evt.Action);
AddParameter(cmd, "@o" + i, SqlDbType.VarChar, size: 16, evt.Outcome.ToString());
AddParameter(cmd, "@c" + i, SqlDbType.VarChar, size: 32, evt.Category);
AddParameter(cmd, "@g" + i, SqlDbType.NVarChar, size: 256, evt.Target);
AddParameter(cmd, "@s" + i, SqlDbType.VarChar, size: 64, evt.SourceNode);
AddParameter(cmd, "@a" + i, SqlDbType.NVarChar, size: 0, actor);
AddParameter(cmd, "@n" + i, SqlDbType.VarChar, size: 0, evt.Action);
AddParameter(cmd, "@o" + i, SqlDbType.VarChar, size: 0, evt.Outcome.ToString());
AddParameter(cmd, "@c" + i, SqlDbType.VarChar, size: 0, evt.Category);
AddParameter(cmd, "@g" + i, SqlDbType.NVarChar, size: 0, evt.Target);
AddParameter(cmd, "@s" + i, SqlDbType.VarChar, size: 0, evt.SourceNode);
AddParameter(cmd, "@r" + i, SqlDbType.UniqueIdentifier, size: 0, evt.CorrelationId);
AddParameter(cmd, "@d" + i, SqlDbType.NVarChar, size: -1, evt.DetailsJson);
}
@@ -290,6 +304,14 @@ VALUES
/// which is what makes the VALUES constructor's derived column types stable
/// regardless of which rows happen to carry nulls.
/// </summary>
/// <remarks>
/// <paramref name="size"/> semantics: <c>0</c> leaves <see cref="SqlParameter.Size"/>
/// unset so SqlClient sizes the parameter from the VALUE — the only safe
/// choice for the variable-length audit columns, because an explicit Size
/// SMALLER than the value silently truncates it at bind time instead of
/// letting the server raise 2628. <c>-1</c> is the explicit MAX marker
/// (nvarchar(max) DetailsJson). Never pass a column width here.
/// </remarks>
private static void AddParameter(
System.Data.Common.DbCommand cmd, string name, SqlDbType type, int size, object? value)
{
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/NotificationOutboxRepository.cs
+20 -3
View File
@@ -158,7 +158,7 @@ VALUES
}
/// <inheritdoc />
public async Task UpdateAsync(Notification n, CancellationToken cancellationToken = default)
public async Task<bool> UpdateAsync(Notification n, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(n);
@@ -184,7 +184,12 @@ VALUES
var notificationId = n.NotificationId;
await _context.Notifications
// The row count is the not-found signal and MUST NOT be discarded.
// DbSet.Update + SaveChanges used to throw DbUpdateConcurrencyException
// when the row had vanished (the daily retention purge deletes terminal
// rows); ExecuteUpdate just reports "0 rows" and returns, which made an
// operator Retry/Discard of a purged notification answer "success".
var rowsAffected = await _context.Notifications
.Where(row => row.NotificationId == notificationId)
.ExecuteUpdateAsync(
setters => setters
@@ -196,11 +201,23 @@ VALUES
.SetProperty(row => row.NextAttemptAt, nextAttemptAt)
.SetProperty(row => row.DeliveredAt, deliveredAt),
cancellationToken);
return rowsAffected > 0;
}
/// <inheritdoc />
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
{
// AsNoTracking + a keyed predicate rather than FindAsync: the write that
// follows this read is ExecuteUpdate, which bypasses the change tracker
// entirely, so tracking the entity bought nothing but a snapshot copy of
// the nvarchar(max) Body/TypeData columns — and left a stale tracked
// instance in the context that a later read of the same id would return
// in preference to the database. FindAsync cannot be made no-tracking.
return await _context.Notifications
.AsNoTracking()
.FirstOrDefaultAsync(row => row.NotificationId == notificationId, cancellationToken);
}
/// <inheritdoc />
public async Task<(IReadOnlyList<Notification> Rows, int TotalCount)> QueryAsync(
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/SiteCallAuditRepository.cs
+137 -72
View File
@@ -1,3 +1,4 @@
using System.Data;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
@@ -72,20 +73,26 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
var idText = siteCall.TrackedOperationId.Value.ToString("D");
var incomingRank = GetRankOrThrow(siteCall.Status);
// ONE round trip, UPDATE-first (WP2.2). The predecessor issued an
// unconditional IF NOT EXISTS … INSERT and THEN a monotonic UPDATE — two
// statements, two round trips, on every single packet, of which the insert
// half was wasted work for every packet after the first (the steady state:
// a cached call emits Submitted → Forwarded → Attempted → terminal, so
// three of four packets hit an existing row).
// ONE round trip, INSERT-first. Both statements ship in a single command
// text, so the round-trip saving of the WP2.2 rewrite is preserved — but
// the ORDER is back to insert-then-update, because UPDATE-first LOSES
// DATA under a concurrent first-write.
//
// The combined batch below runs UPDATE first and inserts only when the
// UPDATE matched nothing AND the row genuinely does not exist. The
// NOT EXISTS re-check is load-bearing: @@ROWCOUNT = 0 is ALSO what a
// monotonic REJECTION looks like (a stale or regressive packet against an
// existing row), and inserting there would resurrect a row the guard just
// refused. Both statements ship in one command text, so this is one
// round trip, not two.
// The UPDATE-first shape was: UPDATE; SET @updated = @@ROWCOUNT;
// IF @updated = 0 AND NOT EXISTS(…) INSERT. Two writers racing the FIRST
// packet of one TrackedOperationId — the cached dual-write and the
// reconciliation pull, which routinely carry DIFFERENT lifecycle states —
// both find no row, so both UPDATEs match nothing. The loser then either
// fails its own NOT EXISTS re-check (READ COMMITTED, after the winner
// committed) and skips the INSERT, or attempts it and eats a 2627 in the
// catch below. EITHER WAY the loser's Status/RetryCount/HttpStatus/
// TerminalAtUtc are silently dropped: it never ran an UPDATE against the
// winner's row.
//
// INSERT-first has no such hole. The loser's INSERT is skipped or faults,
// and the monotonic UPDATE that FOLLOWS it applies its state to whichever
// row won — so the newer lifecycle state survives regardless of
// interleaving, and a stale one is still rejected by the rank guard.
//
// Monotonic update semantics are unchanged: mutate when EITHER the
// incoming rank is strictly greater, OR the incoming rank equals the
@@ -95,7 +102,19 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
// live instead of freezing at the first Attempted packet. Terminal ranks
// are excluded from the tiebreaker, so a later terminal NEVER overwrites
// an earlier one; equal stamps are inert (idempotent replay) and a lower
// rank is always a no-op.
// rank is always a no-op. That inertness is what makes the UPDATE
// harmless immediately after this same call's own INSERT: equal rank,
// equal UpdatedAtUtc, zero rows changed.
//
// Raw SQL with explicitly-typed parameters (rather than
// ExecuteSqlInterpolated) so the monotonic UPDATE exists as ONE statement
// text shared by the combined batch and the duplicate-key retry below —
// the predicate is far too intricate to keep in two copies. Explicit
// SqlDbType is load-bearing: an untyped DateTime parameter binds as
// `datetime` (3.33 ms rounding), which would corrupt both the stored
// datetime2 stamps and the `UpdatedAtUtc <` freshness tiebreaker. Sizes
// are deliberately left at the value's own length so the server enforces
// the column widths (see AuditLogRepository.AddParameter).
//
// SourceNode-stamping: the column is included in the INSERT column list /
// VALUES so a fresh row carries the originating node name (node-a/node-b
@@ -117,57 +136,11 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
// idempotent.
try
{
await _context.Database.ExecuteSqlInterpolatedAsync(
$@"DECLARE @updated int;
UPDATE dbo.SiteCalls
SET Status = {siteCall.Status},
RetryCount = {siteCall.RetryCount},
LastError = {siteCall.LastError},
HttpStatus = {siteCall.HttpStatus},
UpdatedAtUtc = {siteCall.UpdatedAtUtc},
TerminalAtUtc = {siteCall.TerminalAtUtc},
IngestedAtUtc = {siteCall.IngestedAtUtc},
SourceNode = COALESCE({siteCall.SourceNode}, SourceNode)
WHERE TrackedOperationId = {idText}
AND ( {incomingRank} > (CASE Status
WHEN 'Submitted' THEN 0
WHEN 'Forwarded' THEN 1
WHEN 'Attempted' THEN 2
WHEN 'Skipped' THEN 2
WHEN 'Delivered' THEN 3
WHEN 'Failed' THEN 3
WHEN 'Parked' THEN 3
WHEN 'Discarded' THEN 3
ELSE -1
END)
OR ( {incomingRank} = (CASE Status
WHEN 'Submitted' THEN 0
WHEN 'Forwarded' THEN 1
WHEN 'Attempted' THEN 2
WHEN 'Skipped' THEN 2
WHEN 'Delivered' THEN 3
WHEN 'Failed' THEN 3
WHEN 'Parked' THEN 3
WHEN 'Discarded' THEN 3
ELSE -1
END)
AND {incomingRank} < {TerminalRank}
AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );
-- Captured IMMEDIATELY after the UPDATE: @@ROWCOUNT is reset by the next
-- statement, and reading it inline inside a compound IF condition alongside a
-- subquery is not safe (the subquery's own execution can clobber it).
SET @updated = @@ROWCOUNT;
IF @updated = 0 AND NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
INSERT INTO dbo.SiteCalls
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
VALUES
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
ct);
await _context.Database.ExecuteSqlRawAsync(
InsertIfAbsentSql + "\n\n" + MonotonicUpdateSql,
BuildUpsertParameters(siteCall, idText, incomingRank),
ct)
.ConfigureAwait(false);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
@@ -175,20 +148,112 @@ VALUES
{
// Two concurrent sessions both found the row absent and both raced to
// INSERT; the loser raises 2601/2627 against the TrackedOperationId
// primary key. The winner's row IS the first-write, and this packet's
// content is by construction the same lifecycle state, so the race
// outcome is semantically a no-op. Swallow at Debug — the same
// check-then-act contract the sibling AuditLog/Notification repos
// document. Note the loser's UPDATE leg already ran (against no row),
// so nothing is left half-applied.
// primary key. The winner's row IS the first-write, but it is NOT
// necessarily this packet's lifecycle state — the reconciliation pull
// and the cached dual-write feed this method with different states —
// so the loser MUST still apply its monotonic UPDATE against the
// winner's row. The batch aborted at the faulting INSERT, so re-run
// the UPDATE alone here; it is idempotent and rank-guarded, so
// re-running it is safe even if the batch did reach it.
_logger.LogDebug(
ex,
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; treating as no-op.",
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; re-running the monotonic update against the winning row.",
ex.Number,
idText);
await _context.Database.ExecuteSqlRawAsync(
MonotonicUpdateSql,
BuildUpsertParameters(siteCall, idText, incomingRank),
ct)
.ConfigureAwait(false);
}
}
// Leg 1 of the upsert: create the row when it does not exist yet. Runs FIRST
// so a concurrent first-write loser still has a row to update (see UpsertAsync).
private const string InsertIfAbsentSql = @"
IF NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = @Id)
INSERT INTO dbo.SiteCalls
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
VALUES
(@Id, @Channel, @Target, @SourceSite, @SourceNode, @Status, @RetryCount,
@LastError, @HttpStatus, @CreatedAtUtc, @UpdatedAtUtc, @TerminalAtUtc, @IngestedAtUtc);";
// Leg 2 of the upsert: the monotonic guard. Also re-run standalone from the
// duplicate-key catch, which is why it lives in its own constant.
private const string MonotonicUpdateSql = @"
UPDATE dbo.SiteCalls
SET Status = @Status,
RetryCount = @RetryCount,
LastError = @LastError,
HttpStatus = @HttpStatus,
UpdatedAtUtc = @UpdatedAtUtc,
TerminalAtUtc = @TerminalAtUtc,
IngestedAtUtc = @IngestedAtUtc,
SourceNode = COALESCE(@SourceNode, SourceNode)
WHERE TrackedOperationId = @Id
AND ( @Rank > (CASE Status
WHEN 'Submitted' THEN 0
WHEN 'Forwarded' THEN 1
WHEN 'Attempted' THEN 2
WHEN 'Skipped' THEN 2
WHEN 'Delivered' THEN 3
WHEN 'Failed' THEN 3
WHEN 'Parked' THEN 3
WHEN 'Discarded' THEN 3
ELSE -1
END)
OR ( @Rank = (CASE Status
WHEN 'Submitted' THEN 0
WHEN 'Forwarded' THEN 1
WHEN 'Attempted' THEN 2
WHEN 'Skipped' THEN 2
WHEN 'Delivered' THEN 3
WHEN 'Failed' THEN 3
WHEN 'Parked' THEN 3
WHEN 'Discarded' THEN 3
ELSE -1
END)
AND @Rank < @TerminalRank
AND UpdatedAtUtc < @UpdatedAtUtc ) );";
/// <summary>
/// Builds one FRESH parameter set for the upsert statements. Fresh per call
/// because a <see cref="SqlParameter"/> instance cannot be attached to two
/// commands, and the duplicate-key retry issues a second command.
/// </summary>
private static object[] BuildUpsertParameters(SiteCall siteCall, string idText, int incomingRank)
{
return
[
Param("@Id", SqlDbType.VarChar, idText),
Param("@Channel", SqlDbType.VarChar, siteCall.Channel),
Param("@Target", SqlDbType.VarChar, siteCall.Target),
Param("@SourceSite", SqlDbType.VarChar, siteCall.SourceSite),
Param("@SourceNode", SqlDbType.VarChar, siteCall.SourceNode),
Param("@Status", SqlDbType.VarChar, siteCall.Status),
Param("@RetryCount", SqlDbType.Int, siteCall.RetryCount),
Param("@LastError", SqlDbType.NVarChar, siteCall.LastError),
Param("@HttpStatus", SqlDbType.Int, siteCall.HttpStatus),
Param("@CreatedAtUtc", SqlDbType.DateTime2, siteCall.CreatedAtUtc),
Param("@UpdatedAtUtc", SqlDbType.DateTime2, siteCall.UpdatedAtUtc),
Param("@TerminalAtUtc", SqlDbType.DateTime2, siteCall.TerminalAtUtc),
Param("@IngestedAtUtc", SqlDbType.DateTime2, siteCall.IngestedAtUtc),
Param("@Rank", SqlDbType.Int, incomingRank),
Param("@TerminalRank", SqlDbType.Int, TerminalRank),
];
}
/// <summary>
/// Binds one explicitly-typed parameter, mapping a null CLR value to
/// <see cref="DBNull"/> while KEEPING the declared type. Size is never set —
/// SqlClient sizes from the value, so an over-long string is rejected by the
/// server rather than truncated client-side.
/// </summary>
private static SqlParameter Param(string name, SqlDbType type, object? value) =>
new(name, type) { Value = value ?? DBNull.Value };
/// <inheritdoc />
public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default)
{
src/ZB.MOM.WW.ScadaBridge.NotificationOutbox/NotificationOutboxActor.cs
+50 -4
View File
@@ -603,7 +603,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
notification.Status = NotificationStatus.Parked;
notification.LastError = missingAdapterError;
notification.LastAttemptAt = now;
await outboxRepository.UpdateAsync(notification, cancellationToken);
await WarnIfVanishedAsync(outboxRepository, notification, cancellationToken);
await EmitAttemptAuditAsync(
notification,
now,
@@ -654,7 +654,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
break;
}
await outboxRepository.UpdateAsync(notification, cancellationToken);
await WarnIfVanishedAsync(outboxRepository, notification, cancellationToken);
// Emit the per-attempt Attempted row exactly once regardless of the
// outcome (B2). The error message comes from the outcome, not from
@@ -680,6 +680,35 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
}
}
/// <summary>
/// Persists the dispatcher's delivery-state write and logs when the row has
/// VANISHED underneath it — <see cref="INotificationOutboxRepository.UpdateAsync"/>
/// returning false means the retention purge deleted the notification between
/// the claim and the write.
/// </summary>
/// <remarks>
/// The dispatcher deliberately does not treat this as an error: the delivery
/// itself already happened (or failed) and there is no row left to record the
/// outcome on, so there is nothing to retry or roll back. The audit rows are
/// still emitted — they are the durable record. The operator one-shots
/// (retry/discard) take the opposite stance and answer "not found", because a
/// human is waiting on that answer.
/// </remarks>
private async Task WarnIfVanishedAsync(
INotificationOutboxRepository repository,
Notification notification,
CancellationToken cancellationToken)
{
var persisted = await repository.UpdateAsync(notification, cancellationToken);
if (!persisted)
{
_logger.LogWarning(
"Notification {NotificationId} disappeared before its delivery state could be written (status {Status}); the row was most likely purged mid-flight.",
notification.NotificationId,
notification.Status);
}
}
/// <summary>
/// True for <see cref="NotificationStatus.Delivered"/>,
/// <see cref="NotificationStatus.Parked"/>, or
@@ -1098,7 +1127,16 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
notification.RetryCount = 0;
notification.NextAttemptAt = null;
notification.LastError = null;
await repository.UpdateAsync(notification);
// Zero rows updated means the row was purged between the read above and
// this write. Answer the operator honestly instead of reporting a
// re-queue that never happened — and emit no un-park audit row, because
// there is nothing to attribute it to.
if (!await repository.UpdateAsync(notification))
{
return new RetryNotificationResponse(
request.CorrelationId, Success: false, ErrorMessage: "notification not found");
}
// Operator re-queued a parked notification. Emit a Submitted NotifyDeliver
// row attributing the un-park to the operator — otherwise the lifecycle
@@ -1170,7 +1208,15 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
}
notification.Status = NotificationStatus.Discarded;
await repository.UpdateAsync(notification);
// Same vanished-row honesty as the retry path: a purge between the read
// and the write leaves nothing to discard, so report not-found rather
// than success (and skip the terminal audit row below).
if (!await repository.UpdateAsync(notification))
{
return new DiscardNotificationResponse(
request.CorrelationId, Success: false, ErrorMessage: "notification not found");
}
// A manual discard is the OTHER code path that produces
// a terminal NotificationStatus transition (alongside the dispatcher).
src/ZB.MOM.WW.ScadaBridge.SiteCallAudit/SiteCallAuditActor.cs
+88 -2
View File
@@ -221,7 +221,7 @@ public class SiteCallAuditActor : ReceiveActor
ArgumentNullException.ThrowIfNull(repository);
ArgumentNullException.ThrowIfNull(logger);
_injectedRepository = repository;
_injectedRepository = new SerializedRepository(repository);
_logger = logger;
_options = options ?? new SiteCallAuditOptions();
_auditWriter = auditWriter;
@@ -267,7 +267,7 @@ public class SiteCallAuditActor : ReceiveActor
ArgumentNullException.ThrowIfNull(pullClient);
ArgumentNullException.ThrowIfNull(logger);
_injectedRepository = repository;
_injectedRepository = new SerializedRepository(repository);
_siteEnumerator = siteEnumerator;
_pullClient = pullClient;
_logger = logger;
@@ -1564,6 +1564,92 @@ public class SiteCallAuditActor : ReceiveActor
public static readonly PurgeComplete Instance = new();
private PurgeComplete() { }
}
/// <summary>
/// Serializing wrapper applied to a repository handed in through the
/// test constructors. One call at a time, in arrival order.
/// </summary>
/// <remarks>
/// <para>
/// In production every consumer of this actor's repository gets its OWN DI
/// scope — one per message, one per background pass — so the off-mailbox
/// reconciliation and purge passes can safely run alongside mailbox handlers:
/// separate scopes mean separate <c>DbContext</c>s. The injected-repository
/// constructors break that assumption: ONE instance (typically wrapping one
/// <c>DbContext</c>) is shared by the mailbox handlers AND the background
/// passes, and <c>DbContext</c> forbids concurrent operations — "A second
/// operation was started on this context instance" is a hard fault, and the
/// interleaving is nondeterministic, so it surfaces as a flaky test rather
/// than a reliable one.
/// </para>
/// <para>
/// Serializing at the CALL, not around the whole pass, is deliberate: it
/// removes the concurrency hazard while preserving the property the
/// off-mailbox passes exist for — a long drain (blocked in a network pull,
/// holding no repository call) still lets ingest, query and KPI messages be
/// answered. Suspending the mailbox for the duration of a pass would have
/// been simpler and would have invalidated exactly those regression tests.
/// Production is untouched: it never constructs this type.
/// </para>
/// </remarks>
private sealed class SerializedRepository : ISiteCallAuditRepository
{
private readonly ISiteCallAuditRepository _inner;
private readonly SemaphoreSlim _gate = new(1, 1);
public SerializedRepository(ISiteCallAuditRepository inner) => _inner = inner;
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
RunAsync(() => _inner.UpsertAsync(siteCall, ct));
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
RunAsync(() => _inner.GetAsync(id, ct));
public Task<IReadOnlyList<SiteCall>> QueryAsync(
SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
RunAsync(() => _inner.QueryAsync(filter, paging, ct));
public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
RunAsync(() => _inner.PurgeTerminalAsync(olderThanUtc, ct));
public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
DateTime stuckCutoffUtc, DateTime deliveredSinceUtc, CancellationToken ct = default) =>
RunAsync(() => _inner.ComputeKpisAsync(stuckCutoffUtc, deliveredSinceUtc, ct));
public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
DateTime stuckCutoffUtc, DateTime deliveredSinceUtc, CancellationToken ct = default) =>
RunAsync(() => _inner.ComputePerSiteKpisAsync(stuckCutoffUtc, deliveredSinceUtc, ct));
public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
DateTime stuckCutoffUtc, DateTime deliveredSinceUtc, CancellationToken ct = default) =>
RunAsync(() => _inner.ComputePerNodeKpisAsync(stuckCutoffUtc, deliveredSinceUtc, ct));
private async Task RunAsync(Func<Task> call)
{
await _gate.WaitAsync().ConfigureAwait(false);
try
{
await call().ConfigureAwait(false);
}
finally
{
_gate.Release();
}
}
private async Task<T> RunAsync<T>(Func<Task<T>> call)
{
await _gate.WaitAsync().ConfigureAwait(false);
try
{
return await call().ConfigureAwait(false);
}
finally
{
_gate.Release();
}
}
}
}
/// <summary>