dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(config-db): EnableRetryOnFailure fleet-wide + execution-strategy wrap of the combined-telemetry dual-write

Browse Source 
Installs SqlServerRetryingExecutionStrategy (5 retries, 30s cap) so transient SQL
faults retry transparently on every read-side path. Manual transactions aren't
auto-retried, so AuditLogIngestActor's idempotent dual-write is wrapped in an
explicit CreateExecutionStrategy().ExecuteAsync to become retriable. Verified
empirically that existing manual transactions (BundleImporter) are unaffected —
EF skips the strategy while a transaction is already active.
This commit is contained in:
Joseph Doherty
2026-07-09 08:03:30 -04:00
parent 6b06d1efcb
commit 1dd89d523b
4 changed files with 90 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogIngestActor.cs
+39 -24
View File
@@ -1,4 +1,5 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.Audit;
@@ -259,37 +260,51 @@ public class AuditLogIngestActor : ReceiveActor
// skip the registration.
var failureCounter = scope.ServiceProvider.GetService<ICentralAuditWriteFailureCounter>();
// Task 12 (arch-review 04 S5): retry the dual-write under the DbContext's
// configured resiliency strategy. A manual (user-initiated) transaction is
// NOT auto-retried by EnableRetryOnFailure — EF executes directly while a
// transaction is active — so the whole { BEGIN; insert; upsert; COMMIT }
// unit is wrapped here to become a single retriable operation. The unit is
// idempotent (InsertIfNotExists on EventId + monotonic upsert on
// TrackedOperationId), so replay after a rolled-back transient failure is
// safe. NoOpExecutionStrategy on non-configured/test contexts runs it once.
var strategy = dbContext.Database.CreateExecutionStrategy();
foreach (var entry in cmd.Entries)
{
try
{
await using var tx = await dbContext.Database
.BeginTransactionAsync()
.ConfigureAwait(false);
await strategy.ExecuteAsync(async () =>
{
await using var tx = await dbContext.Database
.BeginTransactionAsync()
.ConfigureAwait(false);
// Stamp IngestedAtUtc on both rows from a single
// central-side instant so a join on the two tables sees
// matching timestamps (debugging convenience, not a
// correctness invariant).
var ingestedAt = DateTime.UtcNow;
// Redact the audit half BEFORE the dual-write — only the
// AuditLog row's payload columns are redactable; SiteCalls
// carries operational state only (status, retry count) and
// is left untouched. Null redactor falls back
// to SafeDefault so header redaction always runs.
// IngestedAtUtc is a DetailsJson field
// on the canonical record, so stamp it via the projection helper.
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var filteredAudit = safeRedactor.Apply(entry.Audit);
var auditStamped = AuditRowProjection.WithIngestedAtUtc(filteredAudit, ingestedAt);
var siteCallStamped = entry.SiteCall with { IngestedAtUtc = ingestedAt };
// Stamp IngestedAtUtc on both rows from a single
// central-side instant so a join on the two tables sees
// matching timestamps (debugging convenience, not a
// correctness invariant).
var ingestedAt = DateTime.UtcNow;
// Redact the audit half BEFORE the dual-write — only the
// AuditLog row's payload columns are redactable; SiteCalls
// carries operational state only (status, retry count) and
// is left untouched. Null redactor falls back
// to SafeDefault so header redaction always runs.
// IngestedAtUtc is a DetailsJson field
// on the canonical record, so stamp it via the projection helper.
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var filteredAudit = safeRedactor.Apply(entry.Audit);
var auditStamped = AuditRowProjection.WithIngestedAtUtc(filteredAudit, ingestedAt);
var siteCallStamped = entry.SiteCall with { IngestedAtUtc = ingestedAt };
await auditRepo.InsertIfNotExistsAsync(auditStamped)
.ConfigureAwait(false);
await siteCallRepo.UpsertAsync(siteCallStamped)
.ConfigureAwait(false);
await auditRepo.InsertIfNotExistsAsync(auditStamped)
.ConfigureAwait(false);
await siteCallRepo.UpsertAsync(siteCallStamped)
.ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
accepted.Add(entry.Audit.EventId);
}
catch (Exception ex)