dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(auditlog): reconciliation abandons only row-intrinsic failures and only when a sibling row inserted (review 2026-10-21 follow-up M1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 09:31:26 -04:00
parent 6b8a91cc83
commit 1f87942ce3
4 changed files with 225 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditIngestRejectionTests.cs
+94 -3
View File
@@ -78,15 +78,15 @@ public class AuditIngestRejectionTests : TestKit
private sealed class OneRowClient : IPullAuditEventsClient
{
private readonly AuditEvent _row;
public OneRowClient(AuditEvent row) => _row = row;
private readonly AuditEvent[] _rows;
public OneRowClient(params AuditEvent[] rows) => _rows = rows;
public System.Collections.Concurrent.ConcurrentQueue<DateTime> Since { get; } = new();
public Task<ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse> PullAsync(
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
Since.Enqueue(sinceUtc);
return Task.FromResult(new ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse(
new[] { _row }, MoreAvailable: false));
_rows, MoreAvailable: false));
}
}
@@ -129,4 +129,95 @@ public class AuditIngestRejectionTests : TestKit
Arg.Is<AuditEvent>(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
Arg.Any<CancellationToken>());
}
private SiteAuditReconciliationOptions FastOptions() => new()
{
ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(30),
};
private void RunTicks(IAuditLogRepository repo, OneRowClient client, int ticks)
{
var actor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
new OneSite(), client, StubAuditRepositoryProvider.Build(repo),
Microsoft.Extensions.Options.Options.Create(FastOptions()),
NullLogger<SiteAuditReconciliationActor>.Instance)));
AwaitCondition(() => client.Since.Count >= ticks, TimeSpan.FromSeconds(10));
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor);
}
/// <summary>
/// Follow-up M1: schema drift after a missed central migration (207 invalid column name on
/// every insert) is not a ROW verdict — it never counts toward abandonment, so the site's
/// rows are never flipped Reconciled and purged.
/// </summary>
[Fact]
public void Reconciliation_SchemaDriftOnEveryRow_NeverAbandons()
{
var a = NewEvent();
var b = NewEvent();
var repo = Substitute.For<IAuditLogRepository>();
repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
.ThrowsAsync(SqlExceptionFactory.Create(207));
repo.InsertIfNotExistsAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
.ThrowsAsync(SqlExceptionFactory.Create(207));
var client = new OneRowClient(a, b);
RunTicks(repo, client, 12);
Assert.All(client.Since, s => Assert.Equal(DateTime.MinValue, s));
repo.DidNotReceive().InsertIfNotExistsAsync(
Arg.Is<AuditEvent>(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
Arg.Any<CancellationToken>());
}
/// <summary>
/// Follow-up M1: a ROW-classified error (8114 conversion) that hits EVERY row of the page —
/// e.g. a column whose type a migration changed — is a database problem, not one poison row.
/// With no sibling inserting, nothing is abandoned however many ticks pass.
/// </summary>
[Fact]
public void Reconciliation_PermanentErrorOnEveryRow_NoSiblingSucceeds_NeverAbandons()
{
var a = NewEvent();
var b = NewEvent();
var repo = Substitute.For<IAuditLogRepository>();
repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
.ThrowsAsync(SqlExceptionFactory.Create(8114));
repo.InsertIfNotExistsAsync(Arg.Is<AuditEvent>(e => e.EventId == a.EventId || e.EventId == b.EventId), Arg.Any<CancellationToken>())
.ThrowsAsync(SqlExceptionFactory.Create(8114));
var client = new OneRowClient(a, b);
RunTicks(repo, client, 12);
Assert.All(client.Since, s => Assert.Equal(DateTime.MinValue, s));
repo.DidNotReceive().InsertIfNotExistsAsync(
Arg.Is<AuditEvent>(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
Arg.Any<CancellationToken>());
}
/// <summary>
/// Follow-up M1, positive half: the same row-classified error on ONE row while a sibling in the
/// page inserts IS a poison row — it is abandoned (synthetic row written) after the threshold.
/// </summary>
[Fact]
public void Reconciliation_PermanentErrorOnOneRow_WithSiblingSuccess_Abandons()
{
var ok = NewEvent();
var poison = NewEvent();
var repo = Substitute.For<IAuditLogRepository>();
repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
.ThrowsAsync(SqlExceptionFactory.Create(8114));
repo.InsertIfNotExistsAsync(Arg.Is<AuditEvent>(e => e.EventId == poison.EventId), Arg.Any<CancellationToken>())
.ThrowsAsync(SqlExceptionFactory.Create(8114));
var client = new OneRowClient(ok, poison);
RunTicks(repo, client, 8);
repo.Received().InsertIfNotExistsAsync(
Arg.Is<AuditEvent>(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
Arg.Any<CancellationToken>());
}
}
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/SiteAuditReconciliationActorTests.cs
+30 -7
View File
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
@@ -254,13 +255,29 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
/// <summary>Gets the audit events recorded by this stub.</summary>
public List<AuditEvent> Inserted { get; } = new();
private readonly HashSet<Guid> _seen = new();
private readonly Guid _poisonId;
private readonly Guid _healthyId;
/// <summary>Initializes the stub.</summary>
/// <param name="poisonId">The row that fails with a row-intrinsic (permanent) SQL error.</param>
/// <param name="healthyId">The one non-synthetic row that inserts; every other row fails transiently.</param>
public AlwaysFailUnlessSyntheticRepo(Guid poisonId, Guid healthyId)
{
_poisonId = poisonId;
_healthyId = healthyId;
}
/// <inheritdoc />
public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
{
if (!evt.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal))
if (evt.EventId == _poisonId)
{
throw new InvalidOperationException("simulated permanent insert failure");
throw SqlExceptionFactory.Create(8152);
}
if (evt.EventId != _healthyId
&& !evt.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal))
{
throw new TimeoutException("simulated transient insert failure");
}
if (_seen.Add(evt.EventId))
{
@@ -900,12 +917,15 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
// Five pulls each returning the same poison row so its per-EventId attempt
// counter reaches MaxPermanentInsertAttempts (5); afterwards the scripted
// client returns empty and the actor settles.
// A healthy sibling (earlier timestamp) rides every page: it proves the database
// accepts writes, which is what lets the poison row be abandoned at all (follow-up M1).
// The poison fault is a ROW-intrinsic SQL error (8152 string truncation).
var healthy = NewEvent(siteId, poisonTime.AddSeconds(-1));
var responses = Enumerable.Range(0, 5)
.Select(_ => new PullAuditEventsResponse(new[] { poison }, MoreAvailable: false))
.Select(_ => new PullAuditEventsResponse(new[] { healthy, poison }, MoreAvailable: false))
.ToArray();
var client = new ScriptedPullClient().Script(siteId, responses);
var repo = new PoisonInsertRepo(
poison.EventId, new InvalidOperationException("poison row will never insert"));
var repo = new PoisonInsertRepo(poison.EventId, SqlExceptionFactory.Create(8152));
var actor = CreateActor(sites, client, repo, FastTickOptions());
@@ -965,6 +985,9 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
var siteId = "siteReabandon";
var poisonTime = new DateTime(2026, 5, 20, 10, 5, 0, DateTimeKind.Utc);
var poison = NewEvent(siteId, poisonTime);
// A row that always inserts rides every page so the poison row is abandonable
// (an all-rows-failing page is a database problem and never abandons — follow-up M1).
var healthy = NewEvent(siteId, poisonTime.AddSeconds(-1));
// 10 scripted responses: enough for exactly two 5-attempt abandon
// cycles of the poison row (the cursor never advances between them,
@@ -974,12 +997,12 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
// row count is bounded and deterministic.
var responses = Enumerable.Range(0, 10)
.Select(i => new PullAuditEventsResponse(
new[] { poison, NewEvent(siteId, poisonTime.AddSeconds(i + 1)) },
new[] { healthy, poison, NewEvent(siteId, poisonTime.AddSeconds(i + 1)) },
MoreAvailable: false))
.ToArray();
var client = new ScriptedPullClient().Script(siteId, responses);
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://x:8083"));
var repo = new AlwaysFailUnlessSyntheticRepo();
var repo = new AlwaysFailUnlessSyntheticRepo(poison.EventId, healthy.EventId);
var actor = CreateActor(sites, client, repo, FastTickOptions());