refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)

Solution + 23 src projects + 26 test projects renamed; folders, csproj,
namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated.
ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated.
SQL roles/logins, LDAP domains, CLI command name, and CLI config dir
(~/.scadalink → ~/.scadabridge) also renamed.

Build green; 5 Host.Tests fail awaiting SQL login rename in next commit.
Pre-existing StaleTagMonitor timing flakes unchanged.

Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
Joseph Doherty
2026-05-28 09:37:45 -04:00
parent 6d87ee3c3b
commit 7b0b9c7365
1531 changed files with 11180 additions and 11054 deletions
@@ -0,0 +1,133 @@
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle B (M2-T4) tests for <see cref="FallbackAuditWriter"/> — composes the
/// primary <see cref="SqliteAuditWriter"/>, the drop-oldest
/// <see cref="RingBufferFallback"/>, and an
/// <see cref="IAuditWriteFailureCounter"/> health counter.
/// </summary>
public class FallbackAuditWriterTests
{
private static AuditEvent NewEvent(string? target = null) => new()
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
Target = target,
PayloadTruncated = false,
ForwardState = AuditForwardState.Pending,
};
/// <summary>Flip-switch primary writer mock.</summary>
private sealed class FlipSwitchPrimary : IAuditWriter
{
public bool FailNext { get; set; }
public List<AuditEvent> Written { get; } = new();
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
{
if (FailNext)
{
return Task.FromException(new InvalidOperationException("primary down"));
}
Written.Add(evt);
return Task.CompletedTask;
}
}
[Fact]
public async Task WriteAsync_PrimaryThrows_EventLandsInRing_CallReturnsSuccess()
{
var primary = new FlipSwitchPrimary { FailNext = true };
var ring = new RingBufferFallback(capacity: 16);
var counter = Substitute.For<IAuditWriteFailureCounter>();
var fallback = new FallbackAuditWriter(primary, ring, counter, NullLogger<FallbackAuditWriter>.Instance);
var evt = NewEvent("doomed");
// Must NOT throw — audit failures are always swallowed at this layer.
await fallback.WriteAsync(evt);
Assert.Equal(1, ring.Count);
counter.Received(1).Increment();
}
[Fact]
public async Task WriteAsync_PrimaryRecovers_RingDrains_InFIFOOrder_OnNextWrite()
{
var primary = new FlipSwitchPrimary { FailNext = true };
var ring = new RingBufferFallback(capacity: 16);
var counter = Substitute.For<IAuditWriteFailureCounter>();
var fallback = new FallbackAuditWriter(primary, ring, counter, NullLogger<FallbackAuditWriter>.Instance);
var failed = new[] { NewEvent("a"), NewEvent("b"), NewEvent("c") };
foreach (var e in failed)
{
await fallback.WriteAsync(e);
}
Assert.Equal(3, ring.Count);
// Primary recovers; the very next successful write should drain the
// ring in FIFO order through the primary.
primary.FailNext = false;
var trigger = NewEvent("trigger");
await fallback.WriteAsync(trigger);
Assert.Equal(0, ring.Count);
// Order: the triggering event reaches the primary first (that's the
// signal the primary has recovered), then the backlog drains in FIFO
// submission order behind it.
Assert.Equal(4, primary.Written.Count);
Assert.Equal("trigger", primary.Written[0].Target);
Assert.Equal("a", primary.Written[1].Target);
Assert.Equal("b", primary.Written[2].Target);
Assert.Equal("c", primary.Written[3].Target);
}
[Fact]
public async Task WriteAsync_PrimaryAlwaysSucceeds_Ring_StaysEmpty()
{
var primary = new FlipSwitchPrimary();
var ring = new RingBufferFallback(capacity: 16);
var counter = Substitute.For<IAuditWriteFailureCounter>();
var fallback = new FallbackAuditWriter(primary, ring, counter, NullLogger<FallbackAuditWriter>.Instance);
for (int i = 0; i < 10; i++)
{
await fallback.WriteAsync(NewEvent());
}
Assert.Equal(0, ring.Count);
Assert.Equal(10, primary.Written.Count);
counter.DidNotReceive().Increment();
}
[Fact]
public async Task WriteAsync_FailureCounter_Incremented_Per_PrimaryFailure()
{
var primary = new FlipSwitchPrimary { FailNext = true };
var ring = new RingBufferFallback(capacity: 16);
var counter = Substitute.For<IAuditWriteFailureCounter>();
var fallback = new FallbackAuditWriter(primary, ring, counter, NullLogger<FallbackAuditWriter>.Instance);
for (int i = 0; i < 5; i++)
{
await fallback.WriteAsync(NewEvent());
}
counter.Received(5).Increment();
}
}
@@ -0,0 +1,49 @@
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle C (M5-T7) — the <see cref="HealthMetricsAuditRedactionFailureCounter"/>
/// adapter is the production binding for
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Payload.IAuditRedactionFailureCounter"/> on
/// site nodes; it forwards every <see cref="DefaultAuditPayloadFilter"/>
/// redactor over-redaction event into the shared
/// <see cref="ISiteHealthCollector"/> so the site health report surfaces the
/// count as <c>AuditRedactionFailure</c>. Mirrors the M2 Bundle G
/// HealthMetricsAuditWriteFailureCounter shape one-for-one.
/// </summary>
public class HealthMetricsAuditRedactionFailureCounterTests
{
[Fact]
public void Increment_Routes_To_Collector_IncrementAuditRedactionFailure()
{
var collector = Substitute.For<ISiteHealthCollector>();
var counter = new HealthMetricsAuditRedactionFailureCounter(collector);
counter.Increment();
collector.Received(1).IncrementAuditRedactionFailure();
}
[Fact]
public void Increment_Multiple_Calls_Route_To_Collector_Each_Time()
{
var collector = Substitute.For<ISiteHealthCollector>();
var counter = new HealthMetricsAuditRedactionFailureCounter(collector);
counter.Increment();
counter.Increment();
counter.Increment();
collector.Received(3).IncrementAuditRedactionFailure();
}
[Fact]
public void Construction_With_Null_Collector_Throws_ArgumentNullException()
{
Assert.Throws<ArgumentNullException>(
() => new HealthMetricsAuditRedactionFailureCounter(null!));
}
}
@@ -0,0 +1,46 @@
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle G (M2-T11) — the <see cref="HealthMetricsAuditWriteFailureCounter"/>
/// adapter is the production binding for <see cref="IAuditWriteFailureCounter"/>
/// on site nodes; it forwards every FallbackAuditWriter primary failure into
/// the shared <see cref="ISiteHealthCollector"/> so the site health report
/// surfaces the failure count as <c>SiteAuditWriteFailures</c>.
/// </summary>
public class HealthMetricsAuditWriteFailureCounterTests
{
[Fact]
public void Increment_Routes_To_Collector_IncrementSiteAuditWriteFailures()
{
var collector = Substitute.For<ISiteHealthCollector>();
var counter = new HealthMetricsAuditWriteFailureCounter(collector);
counter.Increment();
collector.Received(1).IncrementSiteAuditWriteFailures();
}
[Fact]
public void Increment_Multiple_Calls_Route_To_Collector_Each_Time()
{
var collector = Substitute.For<ISiteHealthCollector>();
var counter = new HealthMetricsAuditWriteFailureCounter(collector);
counter.Increment();
counter.Increment();
counter.Increment();
collector.Received(3).IncrementSiteAuditWriteFailures();
}
[Fact]
public void Construction_With_Null_Collector_Throws_ArgumentNullException()
{
Assert.Throws<ArgumentNullException>(
() => new HealthMetricsAuditWriteFailureCounter(null!));
}
}
@@ -0,0 +1,91 @@
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle B (M2-T3) tests for <see cref="RingBufferFallback"/> — the
/// drop-oldest fallback used by <see cref="FallbackAuditWriter"/> when the
/// primary SQLite writer is throwing.
/// </summary>
public class RingBufferFallbackTests
{
private static AuditEvent NewEvent(string? target = null)
{
return new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
Target = target,
PayloadTruncated = false,
ForwardState = AuditForwardState.Pending,
};
}
[Fact]
public async Task Enqueue_1025_Into_1024Cap_Ring_DropsOldest_AndRaisesOverflowOnce()
{
var ring = new RingBufferFallback(capacity: 1024);
var overflowCount = 0;
ring.RingBufferOverflowed += () => Interlocked.Increment(ref overflowCount);
var events = Enumerable.Range(0, 1025).Select(i => NewEvent(target: i.ToString())).ToList();
foreach (var e in events)
{
Assert.True(ring.TryEnqueue(e));
}
Assert.Equal(1, overflowCount);
// The surviving 1024 are events[1..1024] (oldest dropped).
var drained = new List<AuditEvent>();
ring.Complete();
await foreach (var e in ring.DrainAsync(CancellationToken.None))
{
drained.Add(e);
}
Assert.Equal(1024, drained.Count);
Assert.Equal("1", drained[0].Target);
Assert.Equal("1024", drained[^1].Target);
}
[Fact]
public async Task DrainAsync_Yields_FIFO_Then_Completes_When_Empty()
{
var ring = new RingBufferFallback(capacity: 16);
var enqueued = Enumerable.Range(0, 5).Select(i => NewEvent(target: i.ToString())).ToList();
foreach (var e in enqueued)
{
Assert.True(ring.TryEnqueue(e));
}
ring.Complete();
var drained = new List<AuditEvent>();
await foreach (var e in ring.DrainAsync(CancellationToken.None))
{
drained.Add(e);
}
Assert.Equal(5, drained.Count);
for (int i = 0; i < 5; i++)
{
Assert.Equal(i.ToString(), drained[i].Target);
}
}
[Fact]
public void TryEnqueue_AllSucceeds_ReturnsTrue()
{
var ring = new RingBufferFallback(capacity: 16);
for (int i = 0; i < 8; i++)
{
Assert.True(ring.TryEnqueue(NewEvent()));
}
}
}
@@ -0,0 +1,191 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle E (M6-T6) tests for <see cref="SqliteAuditWriter.GetBacklogStatsAsync"/>.
/// Exercises the health-metric surface that <c>SiteAuditBacklogReporter</c>
/// polls every 30 s and pushes onto the site health report as
/// <c>SiteAuditBacklog</c>.
/// </summary>
public class SqliteAuditWriterBacklogStatsTests : IDisposable
{
private readonly string _dbPath;
public SqliteAuditWriterBacklogStatsTests()
{
// OnDiskBytes assertions only make sense against a real file — the
// shared-cache in-memory mode returns 0 for the file size, so this
// suite is opinionated about file-backed storage. Tests in
// SqliteAuditWriterWriteTests use in-memory for performance reasons.
_dbPath = Path.Combine(Path.GetTempPath(),
$"audit-backlog-stats-{Guid.NewGuid():N}.db");
}
public void Dispose()
{
if (File.Exists(_dbPath))
{
try { File.Delete(_dbPath); } catch { /* test cleanup best-effort */ }
}
}
private SqliteAuditWriter CreateWriter()
{
var options = new SqliteAuditWriterOptions { DatabasePath = _dbPath };
return new SqliteAuditWriter(
Options.Create(options),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider());
}
private static AuditEvent NewEvent(DateTime? occurredAtUtc = null) => new()
{
EventId = Guid.NewGuid(),
OccurredAtUtc = occurredAtUtc ?? DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
PayloadTruncated = false,
};
[Fact]
public async Task EmptyDb_Returns_Zero_Null_AndZeroBytes()
{
// No file exists yet — the writer ctor creates one but no rows are
// inserted; the snapshot should report a clean queue. OnDiskBytes is
// allowed to be zero (fresh ftruncate) OR small (page header) — the
// contract only requires non-negative; we assert >= 0 and exercise
// the pending fields strictly.
await using var writer = CreateWriter();
var snapshot = await writer.GetBacklogStatsAsync();
Assert.Equal(0, snapshot.PendingCount);
Assert.Null(snapshot.OldestPendingUtc);
Assert.True(snapshot.OnDiskBytes >= 0,
$"OnDiskBytes must be non-negative, got {snapshot.OnDiskBytes}");
}
[Fact]
public async Task Pending_5_Returns_5()
{
await using var writer = CreateWriter();
for (var i = 0; i < 5; i++)
{
await writer.WriteAsync(NewEvent());
}
var snapshot = await writer.GetBacklogStatsAsync();
Assert.Equal(5, snapshot.PendingCount);
}
[Fact]
public async Task OldestPending_Is_Earliest_OccurredAtUtc()
{
await using var writer = CreateWriter();
var t1 = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
var t2 = new DateTime(2026, 5, 20, 10, 1, 0, DateTimeKind.Utc);
var t3 = new DateTime(2026, 5, 20, 10, 2, 0, DateTimeKind.Utc);
// Insert out of order so the snapshot is not "the last write" by
// accident — the OldestPendingUtc must come from a column-min, not
// an insertion-order proxy.
await writer.WriteAsync(NewEvent(t2));
await writer.WriteAsync(NewEvent(t1));
await writer.WriteAsync(NewEvent(t3));
var snapshot = await writer.GetBacklogStatsAsync();
Assert.Equal(3, snapshot.PendingCount);
Assert.NotNull(snapshot.OldestPendingUtc);
// The DB round-trips OccurredAtUtc through the "o" format which
// preserves Kind=Utc — assert tick-equality.
Assert.Equal(t1, snapshot.OldestPendingUtc!.Value);
}
[Fact]
public async Task GetBacklogStatsAsync_DoesNotBlockOnConcurrentWriteLoad()
{
// AuditLog-005: GetBacklogStatsAsync previously took _writeLock, the
// same lock that serialises every batch INSERT in FlushBatch. Under a
// backlog growing to hundreds of thousands of rows a COUNT(*)+MIN
// index scan could park the hot-path writer for hundreds of ms. The
// fix adds a dedicated read-only connection in WAL mode so the probe
// never contends with the writer.
//
// This test demonstrates the lock decoupling by saturating the writer
// with a burst of concurrent writes and asserting that a probe issued
// while those writes are in flight returns inside a tight time bound.
// Without the fix the probe would be queued behind FlushBatch under
// the same _writeLock; with the fix it reads through _readConnection
// and is not gated by the writer.
await using var writer = CreateWriter();
// Seed a baseline so MIN(OccurredAtUtc) has a row to find — the
// important assertion is timing, but a non-empty result also confirms
// the read connection sees the writer's commits via WAL.
for (var i = 0; i < 100; i++)
{
await writer.WriteAsync(NewEvent());
}
// Kick off a sustained write burst on a background task. The writes
// are fire-and-forget — we only need the writer to be busy enough
// that any reuse of _writeLock by the probe would be observable.
var burst = Task.Run(async () =>
{
for (var i = 0; i < 2_000; i++)
{
await writer.WriteAsync(NewEvent()).ConfigureAwait(false);
}
});
// Race the probe against the write burst. The probe must return
// promptly even though the writer is actively flushing batches.
var sw = System.Diagnostics.Stopwatch.StartNew();
var snapshot = await writer.GetBacklogStatsAsync();
sw.Stop();
// Drain the burst before disposing so we don't observe a flake when
// pending writes race with dispose.
await burst;
Assert.True(sw.ElapsedMilliseconds < 1_000,
$"GetBacklogStatsAsync must not block on the writer's _writeLock; took {sw.ElapsedMilliseconds} ms");
Assert.True(snapshot.PendingCount >= 100,
$"backlog probe should see at least the seeded rows; got {snapshot.PendingCount}");
}
[Fact]
public async Task OnDiskBytes_ReturnsFileSize()
{
await using var writer = CreateWriter();
// Insert enough rows to grow the file past the empty schema baseline.
for (var i = 0; i < 100; i++)
{
await writer.WriteAsync(NewEvent());
}
var snapshot = await writer.GetBacklogStatsAsync();
// The exact size depends on SQLite page allocation, but a file-backed
// db with 100 inserted rows MUST be larger than the empty schema
// (a few pages, ~4 KB). The implementation should return the
// FileInfo.Length value verbatim.
Assert.True(File.Exists(_dbPath), $"DB file should exist at {_dbPath}");
var expected = new FileInfo(_dbPath).Length;
Assert.Equal(expected, snapshot.OnDiskBytes);
Assert.True(snapshot.OnDiskBytes > 0,
$"after 100 inserts OnDiskBytes must be > 0, got {snapshot.OnDiskBytes}");
}
}
@@ -0,0 +1,548 @@
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle B (M2-T1) schema-bootstrap tests for <see cref="SqliteAuditWriter"/>.
/// Uses an in-memory shared-cache SQLite database so the same connection name
/// reaches the same file-less db across both the writer and the verifier.
/// </summary>
public class SqliteAuditWriterSchemaTests
{
/// <summary>
/// Each test uses a unique shared-cache in-memory database. The
/// "Mode=Memory;Cache=Shared" syntax lets two SqliteConnections see the same
/// in-memory store as long as both use the same Data Source name.
/// </summary>
private static (SqliteAuditWriter writer, string dataSource) CreateWriter(string testName)
{
var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared";
var options = new SqliteAuditWriterOptions
{
DatabasePath = dataSource,
};
// The writer uses raw "Data Source={path}" by appending Cache=Shared. Override
// by passing the full connection string via the connectionStringOverride hook.
var writer = new SqliteAuditWriter(
Options.Create(options),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride: $"Data Source={dataSource};Cache=Shared");
return (writer, dataSource);
}
private static SqliteConnection OpenVerifierConnection(string dataSource)
{
var connection = new SqliteConnection($"Data Source={dataSource};Cache=Shared");
connection.Open();
return connection;
}
[Fact]
public void Opens_Creates_AuditLog_Table_With_23Columns_And_PK_On_EventId()
{
var (writer, dataSource) = CreateWriter(nameof(Opens_Creates_AuditLog_Table_With_23Columns_And_PK_On_EventId));
using (writer)
{
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "PRAGMA table_info(AuditLog);";
using var reader = cmd.ExecuteReader();
var columns = new List<(string Name, int Pk)>();
while (reader.Read())
{
columns.Add((reader.GetString(1), reader.GetInt32(5)));
}
Assert.Equal(23, columns.Count);
var expected = new[]
{
"EventId", "OccurredAtUtc", "Channel", "Kind", "CorrelationId",
"SourceSiteId", "SourceNode", "SourceInstanceId", "SourceScript", "Actor", "Target",
"Status", "HttpStatus", "DurationMs", "ErrorMessage", "ErrorDetail",
"RequestSummary", "ResponseSummary", "PayloadTruncated", "Extra",
"ForwardState", "ExecutionId", "ParentExecutionId",
};
Assert.Equal(expected.OrderBy(n => n), columns.Select(c => c.Name).OrderBy(n => n));
// PK is EventId only.
var pkColumns = columns.Where(c => c.Pk > 0).Select(c => c.Name).ToList();
Assert.Single(pkColumns);
Assert.Equal("EventId", pkColumns[0]);
}
}
[Fact]
public void Initialize_creates_AuditLog_with_SourceNode_column()
{
var (writer, dataSource) = CreateWriter(nameof(Initialize_creates_AuditLog_with_SourceNode_column));
using (writer)
{
using var connection = OpenVerifierConnection(dataSource);
Assert.True(
ColumnExists(connection, "SourceNode"),
"Fresh AuditLog schema must include the SourceNode column.");
}
}
[Fact]
public void Opens_Creates_IX_ForwardState_Occurred_Index()
{
var (writer, dataSource) = CreateWriter(nameof(Opens_Creates_IX_ForwardState_Occurred_Index));
using (writer)
{
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "PRAGMA index_list(AuditLog);";
using var reader = cmd.ExecuteReader();
var indexNames = new List<string>();
while (reader.Read())
{
indexNames.Add(reader.GetString(1));
}
Assert.Contains("IX_SiteAuditLog_ForwardState_Occurred", indexNames);
// Verify the index columns are ForwardState, OccurredAtUtc in that order.
using var infoCmd = connection.CreateCommand();
infoCmd.CommandText = "PRAGMA index_info(IX_SiteAuditLog_ForwardState_Occurred);";
using var infoReader = infoCmd.ExecuteReader();
var indexColumns = new List<string>();
while (infoReader.Read())
{
indexColumns.Add(infoReader.GetString(2));
}
Assert.Equal(new[] { "ForwardState", "OccurredAtUtc" }, indexColumns);
}
}
[Fact]
public void PRAGMA_auto_vacuum_Is_INCREMENTAL()
{
var (writer, dataSource) = CreateWriter(nameof(PRAGMA_auto_vacuum_Is_INCREMENTAL));
using (writer)
{
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "PRAGMA auto_vacuum;";
var value = Convert.ToInt32(cmd.ExecuteScalar());
// INCREMENTAL = 2 (0 = NONE, 1 = FULL, 2 = INCREMENTAL).
Assert.Equal(2, value);
}
}
// ----- ExecutionId schema-upgrade regression (persistent auditlog.db) ----- //
/// <summary>
/// The OLD pre-ExecutionId-branch <c>AuditLog</c> schema — the 20-column
/// CREATE TABLE WITHOUT the <c>ExecutionId</c> column. A real deployment's
/// on-disk <c>auditlog.db</c> already contains exactly this shape, and
/// <c>CREATE TABLE IF NOT EXISTS</c> is a no-op against it.
/// </summary>
private const string OldPreExecutionIdSchema = """
CREATE TABLE IF NOT EXISTS AuditLog (
EventId TEXT NOT NULL,
OccurredAtUtc TEXT NOT NULL,
Channel TEXT NOT NULL,
Kind TEXT NOT NULL,
CorrelationId TEXT NULL,
SourceSiteId TEXT NULL,
SourceInstanceId TEXT NULL,
SourceScript TEXT NULL,
Actor TEXT NULL,
Target TEXT NULL,
Status TEXT NOT NULL,
HttpStatus INTEGER NULL,
DurationMs INTEGER NULL,
ErrorMessage TEXT NULL,
ErrorDetail TEXT NULL,
RequestSummary TEXT NULL,
ResponseSummary TEXT NULL,
PayloadTruncated INTEGER NOT NULL,
Extra TEXT NULL,
ForwardState TEXT NOT NULL,
PRIMARY KEY (EventId)
);
CREATE INDEX IF NOT EXISTS IX_SiteAuditLog_ForwardState_Occurred
ON AuditLog (ForwardState, OccurredAtUtc);
""";
/// <summary>
/// Seeds a shared-cache in-memory database with the OLD 20-column schema and
/// returns the open connection. The connection MUST stay open for the
/// lifetime of the test: a shared-cache in-memory database is dropped once
/// its last connection closes, so closing this would discard the seeded
/// schema before the writer opens its own connection.
/// </summary>
private static SqliteConnection SeedOldSchemaDatabase(string dataSource)
{
var connection = new SqliteConnection($"Data Source={dataSource};Cache=Shared");
connection.Open();
using var cmd = connection.CreateCommand();
cmd.CommandText = OldPreExecutionIdSchema;
cmd.ExecuteNonQuery();
return connection;
}
private static SqliteAuditWriter CreateWriterOver(string dataSource)
{
var options = new SqliteAuditWriterOptions { DatabasePath = dataSource };
return new SqliteAuditWriter(
Options.Create(options),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride: $"Data Source={dataSource};Cache=Shared");
}
private static bool ColumnExists(SqliteConnection connection, string columnName)
{
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM pragma_table_info('AuditLog') WHERE name = $name";
cmd.Parameters.AddWithValue("$name", columnName);
return Convert.ToInt32(cmd.ExecuteScalar()) > 0;
}
[Fact]
public async Task Opening_Over_PreExisting_OldSchema_Db_Adds_ExecutionId_Column_And_WriteAsync_RoundTrips()
{
var dataSource = $"file:{nameof(Opening_Over_PreExisting_OldSchema_Db_Adds_ExecutionId_Column_And_WriteAsync_RoundTrips)}-{Guid.NewGuid():N}?mode=memory&cache=shared";
// A pre-branch deployment: auditlog.db already exists with the 20-column
// schema and NO ExecutionId column.
using var seedConnection = SeedOldSchemaDatabase(dataSource);
Assert.False(ColumnExists(seedConnection, "ExecutionId"));
// Upgrade: a post-branch SqliteAuditWriter opens the same database. Its
// InitializeSchema must ALTER the missing ExecutionId column in — the
// CREATE TABLE IF NOT EXISTS alone is a no-op against the existing table.
var executionId = Guid.NewGuid();
await using (var writer = CreateWriterOver(dataSource))
{
Assert.True(
ColumnExists(seedConnection, "ExecutionId"),
"SqliteAuditWriter must ALTER the ExecutionId column into a pre-existing AuditLog table.");
// A WriteAsync binding $ExecutionId must now succeed and round-trip;
// without the ALTER it would fail with "no such column: ExecutionId"
// and — because audit writes are best-effort — silently drop the row.
var evt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
PayloadTruncated = false,
ExecutionId = executionId,
};
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal(executionId, row.ExecutionId);
}
// Idempotency: a second writer over the now-upgraded DB must not error
// (the probe sees ExecutionId already present and skips the ALTER).
await using (var writerAgain = CreateWriterOver(dataSource))
{
Assert.True(ColumnExists(seedConnection, "ExecutionId"));
}
}
// ----- ParentExecutionId schema-upgrade regression (persistent auditlog.db) ----- //
/// <summary>
/// The pre-ParentExecutionId-branch <c>AuditLog</c> schema — the 21-column
/// CREATE TABLE that HAS <c>ExecutionId</c> but is WITHOUT
/// <c>ParentExecutionId</c>. A deployment that ran the ExecutionId branch
/// already has an on-disk <c>auditlog.db</c> in exactly this shape, and
/// <c>CREATE TABLE IF NOT EXISTS</c> is a no-op against it.
/// </summary>
private const string OldPreParentExecutionIdSchema = """
CREATE TABLE IF NOT EXISTS AuditLog (
EventId TEXT NOT NULL,
OccurredAtUtc TEXT NOT NULL,
Channel TEXT NOT NULL,
Kind TEXT NOT NULL,
CorrelationId TEXT NULL,
SourceSiteId TEXT NULL,
SourceInstanceId TEXT NULL,
SourceScript TEXT NULL,
Actor TEXT NULL,
Target TEXT NULL,
Status TEXT NOT NULL,
HttpStatus INTEGER NULL,
DurationMs INTEGER NULL,
ErrorMessage TEXT NULL,
ErrorDetail TEXT NULL,
RequestSummary TEXT NULL,
ResponseSummary TEXT NULL,
PayloadTruncated INTEGER NOT NULL,
Extra TEXT NULL,
ForwardState TEXT NOT NULL,
ExecutionId TEXT NULL,
PRIMARY KEY (EventId)
);
CREATE INDEX IF NOT EXISTS IX_SiteAuditLog_ForwardState_Occurred
ON AuditLog (ForwardState, OccurredAtUtc);
""";
/// <summary>
/// Seeds a shared-cache in-memory database with the pre-ParentExecutionId
/// 21-column schema and returns the open connection. The connection MUST
/// stay open for the lifetime of the test — a shared-cache in-memory
/// database is dropped once its last connection closes.
/// </summary>
private static SqliteConnection SeedPreParentExecutionIdSchemaDatabase(string dataSource)
{
var connection = new SqliteConnection($"Data Source={dataSource};Cache=Shared");
connection.Open();
using var cmd = connection.CreateCommand();
cmd.CommandText = OldPreParentExecutionIdSchema;
cmd.ExecuteNonQuery();
return connection;
}
[Fact]
public async Task Opening_Over_PreExisting_PreParentExecutionId_Db_Adds_ParentExecutionId_Column_And_WriteAsync_RoundTrips()
{
var dataSource = $"file:{nameof(Opening_Over_PreExisting_PreParentExecutionId_Db_Adds_ParentExecutionId_Column_And_WriteAsync_RoundTrips)}-{Guid.NewGuid():N}?mode=memory&cache=shared";
// A deployment that ran the ExecutionId branch: auditlog.db already
// exists with the 21-column schema and NO ParentExecutionId column.
using var seedConnection = SeedPreParentExecutionIdSchemaDatabase(dataSource);
Assert.True(ColumnExists(seedConnection, "ExecutionId"));
Assert.False(ColumnExists(seedConnection, "ParentExecutionId"));
// Upgrade: a post-branch SqliteAuditWriter opens the same database. Its
// InitializeSchema must ALTER the missing ParentExecutionId column in —
// the CREATE TABLE IF NOT EXISTS alone is a no-op against the existing
// table.
var executionId = Guid.NewGuid();
var parentExecutionId = Guid.NewGuid();
await using (var writer = CreateWriterOver(dataSource))
{
Assert.True(
ColumnExists(seedConnection, "ParentExecutionId"),
"SqliteAuditWriter must ALTER the ParentExecutionId column into a pre-existing AuditLog table.");
// A WriteAsync binding $ParentExecutionId must now succeed and
// round-trip; without the ALTER it would fail with "no such column:
// ParentExecutionId" and — because audit writes are best-effort —
// silently drop the row.
var evt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
PayloadTruncated = false,
ExecutionId = executionId,
ParentExecutionId = parentExecutionId,
};
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal(executionId, row.ExecutionId);
Assert.Equal(parentExecutionId, row.ParentExecutionId);
}
// Idempotency: a second writer over the now-upgraded DB must not error
// (the probe sees ParentExecutionId already present and skips the ALTER).
await using (var writerAgain = CreateWriterOver(dataSource))
{
Assert.True(ColumnExists(seedConnection, "ParentExecutionId"));
}
}
[Fact]
public async Task WriteAsync_NullParentExecutionId_RoundTripsAsNull()
{
var (writer, _) = CreateWriter(nameof(WriteAsync_NullParentExecutionId_RoundTripsAsNull));
await using (writer)
{
var evt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.Notification,
Kind = AuditKind.NotifySend,
Status = AuditStatus.Submitted,
PayloadTruncated = false,
// ParentExecutionId left null
};
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Null(row.ParentExecutionId);
}
}
// ----- SourceNode schema-upgrade regression (persistent auditlog.db) ----- //
/// <summary>
/// The pre-SourceNode <c>AuditLog</c> schema — the 22-column CREATE TABLE
/// that HAS <c>ExecutionId</c> + <c>ParentExecutionId</c> but is WITHOUT
/// <c>SourceNode</c>. A deployment that ran the ParentExecutionId branch
/// already has an on-disk <c>auditlog.db</c> in exactly this shape, and
/// <c>CREATE TABLE IF NOT EXISTS</c> is a no-op against it.
/// </summary>
private const string OldPreSourceNodeSchema = """
CREATE TABLE IF NOT EXISTS AuditLog (
EventId TEXT NOT NULL,
OccurredAtUtc TEXT NOT NULL,
Channel TEXT NOT NULL,
Kind TEXT NOT NULL,
CorrelationId TEXT NULL,
SourceSiteId TEXT NULL,
SourceInstanceId TEXT NULL,
SourceScript TEXT NULL,
Actor TEXT NULL,
Target TEXT NULL,
Status TEXT NOT NULL,
HttpStatus INTEGER NULL,
DurationMs INTEGER NULL,
ErrorMessage TEXT NULL,
ErrorDetail TEXT NULL,
RequestSummary TEXT NULL,
ResponseSummary TEXT NULL,
PayloadTruncated INTEGER NOT NULL,
Extra TEXT NULL,
ForwardState TEXT NOT NULL,
ExecutionId TEXT NULL,
ParentExecutionId TEXT NULL,
PRIMARY KEY (EventId)
);
CREATE INDEX IF NOT EXISTS IX_SiteAuditLog_ForwardState_Occurred
ON AuditLog (ForwardState, OccurredAtUtc);
""";
/// <summary>
/// Seeds a shared-cache in-memory database with the pre-SourceNode 22-column
/// schema and returns the open connection. The connection MUST stay open for
/// the lifetime of the test — a shared-cache in-memory database is dropped
/// once its last connection closes.
/// </summary>
private static SqliteConnection SeedPreSourceNodeSchemaDatabase(string dataSource)
{
var connection = new SqliteConnection($"Data Source={dataSource};Cache=Shared");
connection.Open();
using var cmd = connection.CreateCommand();
cmd.CommandText = OldPreSourceNodeSchema;
cmd.ExecuteNonQuery();
return connection;
}
[Fact]
public async Task Initialize_adds_SourceNode_to_pre_existing_schema()
{
var dataSource = $"file:{nameof(Initialize_adds_SourceNode_to_pre_existing_schema)}-{Guid.NewGuid():N}?mode=memory&cache=shared";
// A deployment that ran the ParentExecutionId branch: auditlog.db
// already exists with the 22-column schema and NO SourceNode column.
using var seedConnection = SeedPreSourceNodeSchemaDatabase(dataSource);
Assert.True(ColumnExists(seedConnection, "ExecutionId"));
Assert.True(ColumnExists(seedConnection, "ParentExecutionId"));
Assert.False(ColumnExists(seedConnection, "SourceNode"));
// Upgrade: a post-branch SqliteAuditWriter opens the same database. Its
// InitializeSchema must ALTER the missing SourceNode column in — the
// CREATE TABLE IF NOT EXISTS alone is a no-op against the existing table.
await using (var writer = CreateWriterOver(dataSource))
{
Assert.True(
ColumnExists(seedConnection, "SourceNode"),
"SqliteAuditWriter must ALTER the SourceNode column into a pre-existing AuditLog table.");
// A WriteAsync binding $SourceNode must now succeed and round-trip;
// without the ALTER it would fail with "no such column: SourceNode"
// and — because audit writes are best-effort — silently drop the row.
var evt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
PayloadTruncated = false,
SourceNode = "node-a",
};
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal("node-a", row.SourceNode);
}
// Idempotency: a second writer over the now-upgraded DB must not error
// (the probe sees SourceNode already present and skips the ALTER).
await using (var writerAgain = CreateWriterOver(dataSource))
{
Assert.True(ColumnExists(seedConnection, "SourceNode"));
}
}
[Fact]
public async Task WriteAsync_persists_SourceNode_field()
{
var (writer, _) = CreateWriter(nameof(WriteAsync_persists_SourceNode_field));
await using (writer)
{
var evt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
PayloadTruncated = false,
SourceNode = "node-a",
};
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal("node-a", row.SourceNode);
}
}
[Fact]
public async Task WriteAsync_persists_null_SourceNode()
{
var (writer, _) = CreateWriter(nameof(WriteAsync_persists_null_SourceNode));
await using (writer)
{
var evt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.Notification,
Kind = AuditKind.NotifySend,
Status = AuditStatus.Submitted,
PayloadTruncated = false,
// SourceNode left null
};
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Null(row.SourceNode);
}
}
}
@@ -0,0 +1,451 @@
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Bundle B (M2-T2) hot-path tests for <see cref="SqliteAuditWriter"/>. Exercise
/// the Channel-based enqueue, the background writer's batch INSERTs, duplicate-
/// EventId swallowing, ForwardState defaulting, and the
/// <see cref="SqliteAuditWriter.ReadPendingAsync"/> /
/// <see cref="SqliteAuditWriter.MarkForwardedAsync"/> support surface that
/// Bundle D's telemetry actor will call.
/// </summary>
public class SqliteAuditWriterWriteTests
{
private static (SqliteAuditWriter writer, string dataSource) CreateWriter(
string testName,
int? channelCapacity = null,
INodeIdentityProvider? nodeIdentity = null)
{
var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared";
var opts = new SqliteAuditWriterOptions { DatabasePath = dataSource };
if (channelCapacity is int cap)
{
opts.ChannelCapacity = cap;
}
// Default identity provider returns null — existing tests pre-date
// SourceNode stamping and have no expectation about it. New stamping
// tests pass a real provider via the parameter.
var identity = nodeIdentity ?? new FakeNodeIdentityProvider();
var writer = new SqliteAuditWriter(
Options.Create(opts),
NullLogger<SqliteAuditWriter>.Instance,
identity,
connectionStringOverride: $"Data Source={dataSource};Cache=Shared");
return (writer, dataSource);
}
private static SqliteConnection OpenVerifierConnection(string dataSource)
{
var connection = new SqliteConnection($"Data Source={dataSource};Cache=Shared");
connection.Open();
return connection;
}
private static AuditEvent NewEvent(Guid? id = null, DateTime? occurredAtUtc = null)
{
return new AuditEvent
{
EventId = id ?? Guid.NewGuid(),
OccurredAtUtc = occurredAtUtc ?? DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
PayloadTruncated = false,
};
}
[Fact]
public async Task WriteAsync_FreshEvent_PersistsWithForwardStatePending()
{
var (writer, dataSource) = CreateWriter(nameof(WriteAsync_FreshEvent_PersistsWithForwardStatePending));
await using var _ = writer;
var evt = NewEvent();
await writer.WriteAsync(evt);
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT ForwardState FROM AuditLog WHERE EventId = $id;";
cmd.Parameters.AddWithValue("$id", evt.EventId.ToString());
var actual = cmd.ExecuteScalar() as string;
Assert.Equal(AuditForwardState.Pending.ToString(), actual);
}
[Fact]
public async Task WriteAsync_Concurrent_1000Calls_All_Persist_NoExceptions()
{
var (writer, dataSource) = CreateWriter(nameof(WriteAsync_Concurrent_1000Calls_All_Persist_NoExceptions));
await using var _ = writer;
var events = Enumerable.Range(0, 1000).Select(_ => NewEvent()).ToList();
await Parallel.ForEachAsync(events, new ParallelOptions { MaxDegreeOfParallelism = 16 },
async (evt, ct) => await writer.WriteAsync(evt, ct));
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM AuditLog;";
var count = Convert.ToInt64(cmd.ExecuteScalar());
Assert.Equal(1000, count);
}
[Fact]
public async Task WriteAsync_DuplicateEventId_FirstWriteWins_NoException()
{
var (writer, dataSource) = CreateWriter(nameof(WriteAsync_DuplicateEventId_FirstWriteWins_NoException));
await using var _ = writer;
var sharedId = Guid.NewGuid();
var first = NewEvent(sharedId) with { Target = "first" };
var second = NewEvent(sharedId) with { Target = "second" };
await writer.WriteAsync(first);
await writer.WriteAsync(second);
using var connection = OpenVerifierConnection(dataSource);
using var countCmd = connection.CreateCommand();
countCmd.CommandText = "SELECT COUNT(*) FROM AuditLog WHERE EventId = $id;";
countCmd.Parameters.AddWithValue("$id", sharedId.ToString());
var count = Convert.ToInt64(countCmd.ExecuteScalar());
Assert.Equal(1, count);
using var targetCmd = connection.CreateCommand();
targetCmd.CommandText = "SELECT Target FROM AuditLog WHERE EventId = $id;";
targetCmd.Parameters.AddWithValue("$id", sharedId.ToString());
Assert.Equal("first", targetCmd.ExecuteScalar() as string);
}
[Fact]
public async Task WriteAsync_ForcesForwardStatePending_IfNull()
{
var (writer, dataSource) = CreateWriter(nameof(WriteAsync_ForcesForwardStatePending_IfNull));
await using var _ = writer;
var evt = NewEvent() with { ForwardState = null };
await writer.WriteAsync(evt);
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT ForwardState FROM AuditLog WHERE EventId = $id;";
cmd.Parameters.AddWithValue("$id", evt.EventId.ToString());
Assert.Equal(AuditForwardState.Pending.ToString(), cmd.ExecuteScalar() as string);
}
[Fact]
public async Task ReadPendingAsync_Returns_OldestFirst_LimitedToN()
{
var (writer, _) = CreateWriter(nameof(ReadPendingAsync_Returns_OldestFirst_LimitedToN));
await using var _writer = writer;
var baseTime = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var evts = new[]
{
NewEvent(occurredAtUtc: baseTime.AddSeconds(5)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(1)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(3)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(2)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(4)),
};
foreach (var e in evts)
{
await writer.WriteAsync(e);
}
var rows = await writer.ReadPendingAsync(limit: 3);
Assert.Equal(3, rows.Count);
Assert.Equal(baseTime.AddSeconds(1), rows[0].OccurredAtUtc);
Assert.Equal(baseTime.AddSeconds(2), rows[1].OccurredAtUtc);
Assert.Equal(baseTime.AddSeconds(3), rows[2].OccurredAtUtc);
}
[Fact]
public async Task MarkForwardedAsync_FlipsRowsToForwarded()
{
var (writer, dataSource) = CreateWriter(nameof(MarkForwardedAsync_FlipsRowsToForwarded));
await using var _ = writer;
var ids = new[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() };
foreach (var id in ids)
{
await writer.WriteAsync(NewEvent(id));
}
await writer.MarkForwardedAsync(ids);
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT ForwardState, COUNT(*) FROM AuditLog GROUP BY ForwardState;";
using var reader = cmd.ExecuteReader();
var byState = new Dictionary<string, long>();
while (reader.Read())
{
byState[reader.GetString(0)] = reader.GetInt64(1);
}
Assert.Equal(3, byState[AuditForwardState.Forwarded.ToString()]);
Assert.False(byState.ContainsKey(AuditForwardState.Pending.ToString()));
}
[Fact]
public async Task MarkForwardedAsync_NonExistentId_NoThrow()
{
var (writer, _) = CreateWriter(nameof(MarkForwardedAsync_NonExistentId_NoThrow));
await using var _writer = writer;
var phantomIds = new[] { Guid.NewGuid(), Guid.NewGuid() };
await writer.MarkForwardedAsync(phantomIds);
// No assertion needed: the call must complete without throwing.
}
// ----- M6 reconciliation pull surface ----- //
[Fact]
public async Task ReadPendingSinceAsync_Returns_PendingAndForwarded_OldestFirst_LimitedToN()
{
var (writer, dataSource) = CreateWriter(nameof(ReadPendingSinceAsync_Returns_PendingAndForwarded_OldestFirst_LimitedToN));
await using var _ = writer;
var baseTime = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var evts = new[]
{
NewEvent(occurredAtUtc: baseTime.AddSeconds(5)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(1)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(3)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(2)),
NewEvent(occurredAtUtc: baseTime.AddSeconds(4)),
};
foreach (var e in evts) await writer.WriteAsync(e);
// Flip half to Forwarded — they must still surface in the reconciliation pull
// because central hasn't confirmed they were ingested yet.
await writer.MarkForwardedAsync(new[] { evts[0].EventId, evts[2].EventId });
var rows = await writer.ReadPendingSinceAsync(sinceUtc: DateTime.MinValue, batchSize: 3);
Assert.Equal(3, rows.Count);
Assert.Equal(baseTime.AddSeconds(1), rows[0].OccurredAtUtc);
Assert.Equal(baseTime.AddSeconds(2), rows[1].OccurredAtUtc);
Assert.Equal(baseTime.AddSeconds(3), rows[2].OccurredAtUtc);
}
[Fact]
public async Task ReadPendingSinceAsync_ExcludesRowsOlderThanSinceUtc()
{
var (writer, _) = CreateWriter(nameof(ReadPendingSinceAsync_ExcludesRowsOlderThanSinceUtc));
await using var _w = writer;
var baseTime = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var old = NewEvent(occurredAtUtc: baseTime.AddSeconds(-30));
var newer1 = NewEvent(occurredAtUtc: baseTime.AddSeconds(10));
var newer2 = NewEvent(occurredAtUtc: baseTime.AddSeconds(20));
await writer.WriteAsync(old);
await writer.WriteAsync(newer1);
await writer.WriteAsync(newer2);
var rows = await writer.ReadPendingSinceAsync(sinceUtc: baseTime, batchSize: 10);
Assert.Equal(2, rows.Count);
Assert.Contains(rows, r => r.EventId == newer1.EventId);
Assert.Contains(rows, r => r.EventId == newer2.EventId);
Assert.DoesNotContain(rows, r => r.EventId == old.EventId);
}
[Fact]
public async Task ReadPendingSinceAsync_ExcludesReconciledRows()
{
var (writer, _) = CreateWriter(nameof(ReadPendingSinceAsync_ExcludesReconciledRows));
await using var _w = writer;
var baseTime = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var pending = NewEvent(occurredAtUtc: baseTime);
var reconciled = NewEvent(occurredAtUtc: baseTime.AddSeconds(1));
await writer.WriteAsync(pending);
await writer.WriteAsync(reconciled);
await writer.MarkReconciledAsync(new[] { reconciled.EventId });
var rows = await writer.ReadPendingSinceAsync(sinceUtc: DateTime.MinValue, batchSize: 10);
Assert.Single(rows);
Assert.Equal(pending.EventId, rows[0].EventId);
}
[Fact]
public async Task ReadPendingSinceAsync_InvalidBatchSize_Throws()
{
var (writer, _) = CreateWriter(nameof(ReadPendingSinceAsync_InvalidBatchSize_Throws));
await using var _w = writer;
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(
() => writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 0));
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(
() => writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: -3));
}
[Fact]
public async Task MarkReconciledAsync_FlipsPendingAndForwarded_To_Reconciled()
{
var (writer, dataSource) = CreateWriter(nameof(MarkReconciledAsync_FlipsPendingAndForwarded_To_Reconciled));
await using var _ = writer;
var a = NewEvent();
var b = NewEvent();
var c = NewEvent();
await writer.WriteAsync(a);
await writer.WriteAsync(b);
await writer.WriteAsync(c);
// b is currently Forwarded; a and c are Pending.
await writer.MarkForwardedAsync(new[] { b.EventId });
await writer.MarkReconciledAsync(new[] { a.EventId, b.EventId, c.EventId });
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT ForwardState, COUNT(*) FROM AuditLog GROUP BY ForwardState;";
using var reader = cmd.ExecuteReader();
var byState = new Dictionary<string, long>();
while (reader.Read())
{
byState[reader.GetString(0)] = reader.GetInt64(1);
}
Assert.Equal(3, byState[AuditForwardState.Reconciled.ToString()]);
Assert.False(byState.ContainsKey(AuditForwardState.Pending.ToString()));
Assert.False(byState.ContainsKey(AuditForwardState.Forwarded.ToString()));
}
[Fact]
public async Task MarkReconciledAsync_Idempotent_LeavesAlreadyReconciledRowsUntouched()
{
var (writer, dataSource) = CreateWriter(nameof(MarkReconciledAsync_Idempotent_LeavesAlreadyReconciledRowsUntouched));
await using var _ = writer;
var a = NewEvent();
await writer.WriteAsync(a);
await writer.MarkReconciledAsync(new[] { a.EventId });
// Re-call must not throw and must leave the single row Reconciled.
await writer.MarkReconciledAsync(new[] { a.EventId });
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT ForwardState FROM AuditLog WHERE EventId = $id;";
cmd.Parameters.AddWithValue("$id", a.EventId.ToString());
Assert.Equal(AuditForwardState.Reconciled.ToString(), cmd.ExecuteScalar() as string);
}
[Fact]
public async Task MarkReconciledAsync_NonExistentId_NoThrow()
{
var (writer, _) = CreateWriter(nameof(MarkReconciledAsync_NonExistentId_NoThrow));
await using var _w = writer;
await writer.MarkReconciledAsync(new[] { Guid.NewGuid(), Guid.NewGuid() });
// Completes without throwing.
}
// ----- ExecutionId column (universal per-run correlation value) ----- //
[Fact]
public async Task WriteAsync_NonNullExecutionId_RoundTripsThroughMapRow()
{
var (writer, _) = CreateWriter(nameof(WriteAsync_NonNullExecutionId_RoundTripsThroughMapRow));
await using var _w = writer;
var executionId = Guid.NewGuid();
var evt = NewEvent() with { ExecutionId = executionId };
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal(executionId, row.ExecutionId);
}
[Fact]
public async Task WriteAsync_NullExecutionId_RoundTripsAsNull()
{
var (writer, _) = CreateWriter(nameof(WriteAsync_NullExecutionId_RoundTripsAsNull));
await using var _w = writer;
var evt = NewEvent() with { ExecutionId = null };
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Null(row.ExecutionId);
}
// ----- SourceNode stamping (Tasks 11/12) ----- //
[Fact]
public async Task WriteAsync_StampsSourceNodeFromProvider_WhenEventHasNone()
{
var (writer, _) = CreateWriter(
nameof(WriteAsync_StampsSourceNodeFromProvider_WhenEventHasNone),
nodeIdentity: new FakeNodeIdentityProvider("node-a"));
await using var _w = writer;
var evt = NewEvent();
Assert.Null(evt.SourceNode); // sanity check — fresh event has no SourceNode
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal("node-a", row.SourceNode);
}
[Fact]
public async Task WriteAsync_PreservesCallerProvidedSourceNode()
{
var (writer, _) = CreateWriter(
nameof(WriteAsync_PreservesCallerProvidedSourceNode),
nodeIdentity: new FakeNodeIdentityProvider("node-a"));
await using var _w = writer;
// Reconciled rows from another node arrive with their origin's
// SourceNode already populated; the writer must preserve it.
var evt = NewEvent() with { SourceNode = "node-z" };
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Equal("node-z", row.SourceNode);
}
[Fact]
public async Task WriteAsync_LeavesSourceNodeNull_WhenProviderReturnsNull()
{
var (writer, _) = CreateWriter(
nameof(WriteAsync_LeavesSourceNodeNull_WhenProviderReturnsNull),
nodeIdentity: new FakeNodeIdentityProvider(nodeName: null));
await using var _w = writer;
var evt = NewEvent();
await writer.WriteAsync(evt);
var rows = await writer.ReadPendingAsync(limit: 10);
var row = Assert.Single(rows);
Assert.Null(row.SourceNode);
}
}
@@ -0,0 +1,394 @@
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle E Tasks E4/E5 bridge tests. The bridge ingests
/// <see cref="CachedCallAttemptContext"/> notifications from the S&amp;F
/// retry loop and routes them through <see cref="ICachedCallTelemetryForwarder"/>
/// as one or two <see cref="CachedCallTelemetry"/> packets:
/// <list type="bullet">
/// <item><description>Per-attempt: one <c>ApiCallCached</c>/<c>DbWriteCached</c> Attempted row.</description></item>
/// <item><description>Terminal (Delivered/PermanentFailure/ParkedMaxRetries): adds a CachedResolve row carrying the terminal Status.</description></item>
/// </list>
/// </summary>
public class CachedCallLifecycleBridgeTests
{
private readonly ICachedCallTelemetryForwarder _forwarder = Substitute.For<ICachedCallTelemetryForwarder>();
private readonly TrackedOperationId _id = TrackedOperationId.New();
private CachedCallLifecycleBridge CreateSut() => new(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance);
private CachedCallAttemptContext Ctx(
CachedCallAttemptOutcome outcome,
string channel = "ApiOutbound",
int retryCount = 1,
string? lastError = null,
int? httpStatus = null,
Guid? executionId = null,
string? sourceScript = null,
Guid? parentExecutionId = null) =>
new(
TrackedOperationId: _id,
Channel: channel,
Target: "ERP.GetOrder",
SourceSite: "site-77",
Outcome: outcome,
RetryCount: retryCount,
LastError: lastError,
HttpStatus: httpStatus,
CreatedAtUtc: new DateTime(2026, 5, 20, 9, 0, 0, DateTimeKind.Utc),
OccurredAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
DurationMs: 42,
SourceInstanceId: "Plant.Pump42",
ExecutionId: executionId,
SourceScript: sourceScript,
ParentExecutionId: parentExecutionId);
[Fact]
public async Task TransientFailure_EmitsOneAttemptedRow_NoResolve()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
retryCount: 2,
lastError: "HTTP 503",
httpStatus: 503));
var packet = Assert.Single(captured);
Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind);
Assert.Equal(AuditStatus.Attempted, packet.Audit.Status);
Assert.Equal(503, packet.Audit.HttpStatus);
Assert.Equal("HTTP 503", packet.Audit.ErrorMessage);
Assert.Equal(_id.Value, packet.Audit.CorrelationId);
Assert.Equal("Attempted", packet.Operational.Status);
Assert.Equal(2, packet.Operational.RetryCount);
Assert.Null(packet.Operational.TerminalAtUtc);
}
[Fact]
public async Task Delivered_EmitsAttemptedRow_AndCachedResolveDelivered()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
Assert.Equal(2, captured.Count);
var attempted = captured[0];
Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind);
Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status);
Assert.Equal("Attempted", attempted.Operational.Status);
Assert.Null(attempted.Operational.TerminalAtUtc);
var resolve = captured[1];
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind);
Assert.Equal(AuditStatus.Delivered, resolve.Audit.Status);
Assert.Equal("Delivered", resolve.Operational.Status);
Assert.NotNull(resolve.Operational.TerminalAtUtc);
Assert.Equal(_id.Value, resolve.Audit.CorrelationId);
}
[Fact]
public async Task PermanentFailure_EmitsAttempted_AndCachedResolveParked()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.PermanentFailure,
lastError: "Permanent failure (handler returned false)"));
Assert.Equal(2, captured.Count);
Assert.Equal(AuditKind.ApiCallCached, captured[0].Audit.Kind);
Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind);
Assert.Equal(AuditStatus.Parked, captured[1].Audit.Status);
Assert.Equal("Parked", captured[1].Operational.Status);
}
[Fact]
public async Task ParkedMaxRetries_EmitsAttempted_AndCachedResolveParked()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.ParkedMaxRetries));
Assert.Equal(2, captured.Count);
Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind);
Assert.Equal(AuditStatus.Parked, captured[1].Audit.Status);
}
[Fact]
public async Task DbChannel_MapsToDbWriteCachedKind_AndDbOutboundChannel()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.Delivered, channel: "DbOutbound"));
Assert.Equal(2, captured.Count);
Assert.Equal(AuditKind.DbWriteCached, captured[0].Audit.Kind);
Assert.Equal(AuditChannel.DbOutbound, captured[0].Audit.Channel);
Assert.Equal("DbOutbound", captured[0].Operational.Channel);
Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind);
Assert.Equal(AuditChannel.DbOutbound, captured[1].Audit.Channel);
}
[Fact]
public async Task BridgeDoesNotThrow_WhenForwarderThrows()
{
_forwarder
.ForwardAsync(Arg.Any<CachedCallTelemetry>(), Arg.Any<CancellationToken>())
.Returns(Task.FromException(new InvalidOperationException("forwarder down")));
var sut = CreateSut();
// Must not throw — best-effort emission.
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
}
[Fact]
public async Task BridgePopulatesProvenance_FromAttemptContext()
{
CachedCallTelemetry? captured = null;
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured = t), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
retryCount: 3,
lastError: "transient",
httpStatus: 500));
Assert.NotNull(captured);
Assert.Equal("site-77", captured!.Audit.SourceSiteId);
Assert.Equal("Plant.Pump42", captured.Audit.SourceInstanceId);
Assert.Equal("ERP.GetOrder", captured.Audit.Target);
Assert.Equal(42, captured.Audit.DurationMs);
Assert.Equal(_id.Value, captured.Audit.CorrelationId);
}
// ── Audit Log #23 (ExecutionId Task 4): ExecutionId / SourceScript ──
[Fact]
public async Task RetryLoopAttemptedRow_CarriesExecutionIdAndSourceScript_FromContext()
{
// Task 4: the ExecutionId + SourceScript threaded through the S&F
// buffer arrive on the CachedCallAttemptContext; the bridge must stamp
// both onto the per-attempt ApiCallCached row (previously SourceScript
// was hard-coded null with a "not threaded through S&F" comment).
var executionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
executionId: executionId,
sourceScript: "Plant.Pump42/OnTick"));
var packet = Assert.Single(captured);
Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind);
Assert.Equal(executionId, packet.Audit.ExecutionId);
Assert.Equal("Plant.Pump42/OnTick", packet.Audit.SourceScript);
}
[Fact]
public async Task RetryLoopCachedResolveRow_CarriesExecutionIdAndSourceScript_FromContext()
{
// The terminal CachedResolve row must also carry the threaded
// provenance so the whole retry-loop lifecycle is correlated.
var executionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.Delivered,
channel: "DbOutbound",
executionId: executionId,
sourceScript: "Plant.Tank/OnAlarm"));
Assert.Equal(2, captured.Count);
var resolve = Assert.Single(captured, p => p.Audit.Kind == AuditKind.CachedResolve);
Assert.Equal(executionId, resolve.Audit.ExecutionId);
Assert.Equal("Plant.Tank/OnAlarm", resolve.Audit.SourceScript);
var attempted = Assert.Single(captured, p => p.Audit.Kind == AuditKind.DbWriteCached);
Assert.Equal(executionId, attempted.Audit.ExecutionId);
Assert.Equal("Plant.Tank/OnAlarm", attempted.Audit.SourceScript);
}
[Fact]
public async Task RetryLoopRow_NullExecutionIdAndSourceScript_RemainNull()
{
// Back-compat: a pre-Task-4 buffered row has no ExecutionId /
// SourceScript; the bridge must leave the audit row's fields null
// rather than throwing.
CachedCallTelemetry? captured = null;
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured = t), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
Assert.NotNull(captured);
Assert.Null(captured!.Audit.ExecutionId);
Assert.Null(captured.Audit.SourceScript);
}
// ── Audit Log #23 (ParentExecutionId Task 6): ParentExecutionId ──
[Fact]
public async Task RetryLoopAttemptedRow_CarriesParentExecutionId_FromContext()
{
// Task 6: the ParentExecutionId threaded through the S&F buffer (the
// inbound-API run that spawned the originating script) arrives on the
// CachedCallAttemptContext; the bridge must stamp it onto the
// per-attempt ApiCallCached row beside ExecutionId.
var parentExecutionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
parentExecutionId: parentExecutionId));
var packet = Assert.Single(captured);
Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind);
Assert.Equal(parentExecutionId, packet.Audit.ParentExecutionId);
}
[Fact]
public async Task RetryLoopCachedResolveRow_CarriesParentExecutionId_FromContext()
{
// The terminal CachedResolve row must also carry the threaded
// ParentExecutionId so the whole retry-loop lifecycle correlates back
// to the spawning inbound-API execution.
var parentExecutionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.Delivered,
channel: "DbOutbound",
parentExecutionId: parentExecutionId));
Assert.Equal(2, captured.Count);
var resolve = Assert.Single(captured, p => p.Audit.Kind == AuditKind.CachedResolve);
Assert.Equal(parentExecutionId, resolve.Audit.ParentExecutionId);
var attempted = Assert.Single(captured, p => p.Audit.Kind == AuditKind.DbWriteCached);
Assert.Equal(parentExecutionId, attempted.Audit.ParentExecutionId);
}
[Fact]
public async Task RetryLoopRow_NullParentExecutionId_RemainsNull()
{
// Back-compat / non-routed run: the originating script was not spawned
// by an inbound-API request, so ParentExecutionId is null; the bridge
// must leave the audit row's ParentExecutionId null rather than throwing.
CachedCallTelemetry? captured = null;
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured = t), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
Assert.NotNull(captured);
Assert.Null(captured!.Audit.ParentExecutionId);
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task RetryLoopRow_StampsSourceNode_FromNodeIdentityProvider()
{
// SourceNode-stamping (Task 14): when an INodeIdentityProvider is
// wired the bridge stamps the local node name (node-a/node-b) onto
// the SiteCallOperational.SourceNode column of every emitted packet.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns("node-a");
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = new CachedCallLifecycleBridge(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance, nodeIdentity);
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
Assert.Equal(2, captured.Count);
Assert.All(captured, p => Assert.Equal("node-a", p.Operational.SourceNode));
}
[Fact]
public async Task RetryLoopRow_NoNodeIdentityProvider_LeavesSourceNodeNull()
{
// When no INodeIdentityProvider is wired (legacy hosts / tests) the
// bridge degrades to a null SourceNode rather than throwing. The
// emitted packet's SourceNode is null so the central row persists NULL.
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
// Default CreateSut() does NOT pass a node-identity provider.
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
var packet = Assert.Single(captured);
Assert.Null(packet.Operational.SourceNode);
}
[Fact]
public async Task RetryLoopRow_NodeIdentityWithNullNodeName_LeavesSourceNodeNull()
{
// The provider exists but reports a null NodeName (unconfigured). The
// bridge must pass that null through to SourceNode rather than
// falling back to a placeholder.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns((string?)null);
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = new CachedCallLifecycleBridge(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance, nodeIdentity);
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
var packet = Assert.Single(captured);
Assert.Null(packet.Operational.SourceNode);
}
}
@@ -0,0 +1,307 @@
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle E E2 tests for <see cref="CachedCallTelemetryForwarder"/>. The
/// forwarder is the site-side dual emitter: every cached-call lifecycle event
/// writes one <see cref="AuditEvent"/> to <see cref="IAuditWriter"/> and one
/// operational tracking-row mutation to <see cref="IOperationTrackingStore"/>.
/// Audit-emission contract: best-effort — a thrown writer or tracking store
/// must be logged and swallowed; the forwarder must never propagate the
/// exception to the calling script.
/// </summary>
public class CachedCallTelemetryForwarderTests
{
private readonly IAuditWriter _writer = Substitute.For<IAuditWriter>();
private readonly IOperationTrackingStore _tracking = Substitute.For<IOperationTrackingStore>();
private readonly TrackedOperationId _id = TrackedOperationId.New();
private readonly DateTime _now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
private CachedCallTelemetryForwarder CreateSut() => new(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance);
private CachedCallTelemetry SubmitPacket() =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = _now,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.CachedSubmit,
CorrelationId = _id.Value,
SourceSiteId = "site-1",
SourceInstanceId = "inst-1",
SourceScript = "ScriptActor:doStuff",
Target = "ERP.GetOrder",
Status = AuditStatus.Submitted,
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: _id,
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: "site-1",
SourceNode: null,
Status: "Submitted",
RetryCount: 0,
LastError: null,
HttpStatus: null,
CreatedAtUtc: _now,
UpdatedAtUtc: _now,
TerminalAtUtc: null));
private CachedCallTelemetry AttemptedPacket(int retryCount = 1, string? lastError = "HTTP 500", int? httpStatus = 500) =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = _now,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCallCached,
CorrelationId = _id.Value,
SourceSiteId = "site-1",
Target = "ERP.GetOrder",
Status = AuditStatus.Attempted,
HttpStatus = httpStatus,
ErrorMessage = lastError,
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: _id,
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: "site-1",
SourceNode: null,
Status: "Attempted",
RetryCount: retryCount,
LastError: lastError,
HttpStatus: httpStatus,
CreatedAtUtc: _now,
UpdatedAtUtc: _now,
TerminalAtUtc: null));
private CachedCallTelemetry ResolvePacket(string status = "Delivered") =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = _now,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.CachedResolve,
CorrelationId = _id.Value,
SourceSiteId = "site-1",
Target = "ERP.GetOrder",
Status = Enum.Parse<AuditStatus>(status),
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: _id,
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: "site-1",
SourceNode: null,
Status: status,
RetryCount: 2,
LastError: null,
HttpStatus: null,
CreatedAtUtc: _now,
UpdatedAtUtc: _now,
TerminalAtUtc: _now));
[Fact]
public async Task ForwardAsync_Submit_WritesAuditEvent_AndRecordsEnqueue()
{
var sut = CreateSut();
var packet = SubmitPacket();
await sut.ForwardAsync(packet, CancellationToken.None);
// Audit row: one WriteAsync of the submit event.
await _writer.Received(1).WriteAsync(
Arg.Is<AuditEvent>(e =>
e.EventId == packet.Audit.EventId
&& e.Kind == AuditKind.CachedSubmit
&& e.Status == AuditStatus.Submitted),
Arg.Any<CancellationToken>());
// Tracking row: insert-if-not-exists with kind discriminator.
// Default CreateSut() does NOT supply an INodeIdentityProvider, so the
// forwarder passes null sourceNode to RecordEnqueueAsync (legacy / test
// host behaviour). The Task 14 stamping path is covered by the
// ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider test
// below.
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
null,
Arg.Any<CancellationToken>());
await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync(
default, default!, default, default, default, default);
await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync(
default, default!, default, default, default);
}
[Fact]
public async Task ForwardAsync_Attempted_WritesAuditEvent_AndRecordsAttempt()
{
var sut = CreateSut();
var packet = AttemptedPacket(retryCount: 2, lastError: "HTTP 503", httpStatus: 503);
await sut.ForwardAsync(packet, CancellationToken.None);
await _writer.Received(1).WriteAsync(
Arg.Is<AuditEvent>(e =>
e.EventId == packet.Audit.EventId
&& e.Kind == AuditKind.ApiCallCached
&& e.Status == AuditStatus.Attempted),
Arg.Any<CancellationToken>());
await _tracking.Received(1).RecordAttemptAsync(
_id, "Attempted", 2, "HTTP 503", 503, Arg.Any<CancellationToken>());
await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync(
default, default!, default, default, default, default, default);
await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync(
default, default!, default, default, default);
}
[Fact]
public async Task ForwardAsync_Resolve_WritesAuditEvent_AndRecordsTerminal()
{
var sut = CreateSut();
var packet = ResolvePacket("Delivered");
await sut.ForwardAsync(packet, CancellationToken.None);
await _writer.Received(1).WriteAsync(
Arg.Is<AuditEvent>(e =>
e.EventId == packet.Audit.EventId
&& e.Kind == AuditKind.CachedResolve
&& e.Status == AuditStatus.Delivered),
Arg.Any<CancellationToken>());
await _tracking.Received(1).RecordTerminalAsync(
_id, "Delivered", null, null, Arg.Any<CancellationToken>());
await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync(
default, default!, default, default, default, default, default);
await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync(
default, default!, default, default, default, default);
}
[Fact]
public async Task ForwardAsync_WriterThrows_Logs_DoesNotPropagate()
{
_writer.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
.Throws(new InvalidOperationException("primary down"));
var sut = CreateSut();
// Must not throw.
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
// Tracking still attempted — emission of the two halves is independent
// so a writer outage cannot starve the operational row (and vice-versa).
await _tracking.Received(1).RecordEnqueueAsync(
Arg.Any<TrackedOperationId>(),
Arg.Any<string>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_TrackingStoreThrows_Logs_DoesNotPropagate()
{
_tracking.RecordEnqueueAsync(
Arg.Any<TrackedOperationId>(),
Arg.Any<string>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<CancellationToken>())
.Throws(new InvalidOperationException("sqlite locked"));
var sut = CreateSut();
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
// Writer still attempted — emission halves are independent.
await _writer.Received(1).WriteAsync(
Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_NullPacket_Throws()
{
var sut = CreateSut();
await Assert.ThrowsAsync<ArgumentNullException>(
() => sut.ForwardAsync(null!, CancellationToken.None));
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider()
{
// SourceNode-stamping (Task 14): when an INodeIdentityProvider is
// wired the forwarder must stamp its NodeName onto the
// RecordEnqueueAsync sourceNode parameter so the tracking row
// captures the originating node (node-a/node-b).
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns("node-a");
var sut = new CachedCallTelemetryForwarder(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
"node-a",
Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_Submit_NodeIdentityNullNodeName_PassesNullSourceNode()
{
// The provider exists but reports a null NodeName (unconfigured).
// The forwarder passes that null through to RecordEnqueueAsync rather
// than falling back to a placeholder string.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns((string?)null);
var sut = new CachedCallTelemetryForwarder(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
null,
Arg.Any<CancellationToken>());
}
}
@@ -0,0 +1,201 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Tests for <see cref="ClusterClientSiteAuditClient"/> — the production
/// <see cref="ISiteStreamAuditClient"/> binding wired by the Host for site
/// roles. The client maps the proto-DTO batches produced by
/// <see cref="SiteAuditTelemetryActor"/> into the Akka
/// <see cref="IngestAuditEventsCommand"/> / <see cref="IngestCachedTelemetryCommand"/>
/// messages, Asks the site's <c>SiteCommunicationActor</c> (which forwards over
/// ClusterClient to central), and maps the reply back into an
/// <see cref="IngestAck"/>.
/// </summary>
/// <remarks>
/// A <see cref="TestProbe"/> stands in for the <c>SiteCommunicationActor</c>:
/// it lets the tests assert the exact command shape AND drive the reply (or
/// withhold one to exercise the Ask-timeout path).
/// </remarks>
public class ClusterClientSiteAuditClientTests : TestKit
{
/// <summary>Short Ask timeout so the timeout test completes quickly.</summary>
private static readonly TimeSpan AskTimeout = TimeSpan.FromMilliseconds(500);
private static AuditEvent NewEvent(Guid? id = null) => new()
{
EventId = id ?? Guid.NewGuid(),
OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = "site-1",
ForwardState = AuditForwardState.Pending,
};
private static AuditEventBatch BatchOf(IEnumerable<AuditEvent> events)
{
var batch = new AuditEventBatch();
foreach (var e in events)
{
batch.Events.Add(AuditEventDtoMapper.ToDto(e));
}
return batch;
}
private static SiteCallOperationalDto NewOperationalDto() => new()
{
TrackedOperationId = Guid.NewGuid().ToString(),
Channel = "ApiOutbound",
Target = "ext-system-1",
SourceSite = "site-1",
Status = "Submitted",
RetryCount = 0,
LastError = string.Empty,
CreatedAtUtc = Timestamp.FromDateTime(new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc)),
UpdatedAtUtc = Timestamp.FromDateTime(new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc)),
};
[Fact]
public async Task IngestAuditEventsAsync_FullAck_MapsAllAcceptedIdsOntoAck()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var events = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
var batch = BatchOf(events);
var task = sut.IngestAuditEventsAsync(batch, CancellationToken.None);
// The probe receives exactly one IngestAuditEventsCommand carrying the
// batch's events; it replies with every EventId accepted.
var cmd = probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
Assert.Equal(3, cmd.Events.Count);
Assert.Equal(
events.Select(e => e.EventId).ToHashSet(),
cmd.Events.Select(e => e.EventId).ToHashSet());
probe.Reply(new IngestAuditEventsReply(events.Select(e => e.EventId).ToList()));
var ack = await task;
Assert.Equal(
events.Select(e => e.EventId.ToString()).ToHashSet(),
ack.AcceptedEventIds.ToHashSet());
}
[Fact]
public async Task IngestAuditEventsAsync_PartialAck_OnlyAcceptedIdsAppearOnAck()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var events = Enumerable.Range(0, 5).Select(_ => NewEvent()).ToList();
var accepted = events.Take(3).Select(e => e.EventId).ToList();
var task = sut.IngestAuditEventsAsync(BatchOf(events), CancellationToken.None);
probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
probe.Reply(new IngestAuditEventsReply(accepted));
var ack = await task;
Assert.Equal(3, ack.AcceptedEventIds.Count);
Assert.Equal(
accepted.Select(id => id.ToString()).ToHashSet(),
ack.AcceptedEventIds.ToHashSet());
}
[Fact]
public async Task IngestAuditEventsAsync_AskTimeout_Throws_SoDrainLoopKeepsRowsPending()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var batch = BatchOf(new[] { NewEvent() });
// The probe receives the command but never replies — the Ask times out.
// The contract: a timeout MUST surface as a thrown exception so the
// SiteAuditTelemetryActor drain loop leaves the rows Pending.
var task = sut.IngestAuditEventsAsync(batch, CancellationToken.None);
probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
await Assert.ThrowsAnyAsync<Exception>(() => task);
}
[Fact]
public async Task IngestAuditEventsAsync_FaultedReply_Throws()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var task = sut.IngestAuditEventsAsync(BatchOf(new[] { NewEvent() }), CancellationToken.None);
probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
// A Status.Failure from central (Task 1: central does not swallow an
// ingest fault into an empty ack) must propagate as a thrown exception.
probe.Reply(new Status.Failure(new InvalidOperationException("central ingest faulted")));
await Assert.ThrowsAnyAsync<Exception>(() => task);
}
[Fact]
public async Task IngestCachedTelemetryAsync_RoutesCommand_AndMapsReply()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var events = Enumerable.Range(0, 2).Select(_ => NewEvent()).ToList();
var batch = new CachedTelemetryBatch();
foreach (var e in events)
{
batch.Packets.Add(new CachedTelemetryPacket
{
AuditEvent = AuditEventDtoMapper.ToDto(e),
Operational = NewOperationalDto(),
});
}
var task = sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
// The probe receives an IngestCachedTelemetryCommand (NOT an
// IngestAuditEventsCommand) with one entry per packet.
var cmd = probe.ExpectMsg<IngestCachedTelemetryCommand>(TimeSpan.FromSeconds(3));
Assert.Equal(2, cmd.Entries.Count);
Assert.Equal(
events.Select(e => e.EventId).ToHashSet(),
cmd.Entries.Select(en => en.Audit.EventId).ToHashSet());
probe.Reply(new IngestCachedTelemetryReply(events.Select(e => e.EventId).ToList()));
var ack = await task;
Assert.Equal(
events.Select(e => e.EventId.ToString()).ToHashSet(),
ack.AcceptedEventIds.ToHashSet());
}
[Fact]
public async Task IngestCachedTelemetryAsync_AskTimeout_Throws()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var batch = new CachedTelemetryBatch();
batch.Packets.Add(new CachedTelemetryPacket
{
AuditEvent = AuditEventDtoMapper.ToDto(NewEvent()),
Operational = NewOperationalDto(),
});
var task = sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
probe.ExpectMsg<IngestCachedTelemetryCommand>(TimeSpan.FromSeconds(3));
await Assert.ThrowsAnyAsync<Exception>(() => task);
}
}
@@ -0,0 +1,58 @@
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle E E1 tests for <see cref="NoOpSiteStreamAuditClient"/>. The NoOp
/// client is the default <see cref="ISiteStreamAuditClient"/> binding until M6
/// delivers the gRPC-backed implementation; both <c>IngestAuditEventsAsync</c>
/// (M2) and <c>IngestCachedTelemetryAsync</c> (M3) must return an empty ack
/// (no rows flipped to Forwarded) without throwing or partially handling the
/// batch.
/// </summary>
public class NoOpSiteStreamAuditClientTests
{
[Fact]
public async Task IngestCachedTelemetryAsync_EmptyBatch_ReturnsEmptyAck()
{
var sut = new NoOpSiteStreamAuditClient();
var batch = new CachedTelemetryBatch();
var ack = await sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
Assert.NotNull(ack);
Assert.Empty(ack.AcceptedEventIds);
}
[Fact]
public async Task IngestCachedTelemetryAsync_PopulatedBatch_ReturnsEmptyAck()
{
var sut = new NoOpSiteStreamAuditClient();
var batch = new CachedTelemetryBatch();
batch.Packets.Add(new CachedTelemetryPacket
{
AuditEvent = new AuditEventDto
{
EventId = Guid.NewGuid().ToString(),
Channel = "ApiOutbound",
Kind = "CachedSubmit",
Status = "Submitted",
},
});
var ack = await sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
// No EventIds flipped — NoOp does not forward to anyone.
Assert.Empty(ack.AcceptedEventIds);
}
[Fact]
public async Task IngestCachedTelemetryAsync_NullBatch_Throws()
{
var sut = new NoOpSiteStreamAuditClient();
await Assert.ThrowsAsync<ArgumentNullException>(
() => sut.IngestCachedTelemetryAsync(null!, CancellationToken.None));
}
}
@@ -0,0 +1,456 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle D D1 tests for <see cref="SiteAuditTelemetryActor"/>. The actor drains
/// the site SQLite queue via <see cref="ISiteAuditQueue"/>, pushes batches via
/// <see cref="ISiteStreamAuditClient"/>, and flips ack'd rows to Forwarded.
/// Both collaborators are NSubstitute mocks so the tests never touch real
/// SQLite or gRPC.
/// </summary>
public class SiteAuditTelemetryActorTests : TestKit
{
private readonly ISiteAuditQueue _queue = Substitute.For<ISiteAuditQueue>();
private readonly ISiteStreamAuditClient _client = Substitute.For<ISiteStreamAuditClient>();
private readonly IOperationTrackingStore _trackingStore = Substitute.For<IOperationTrackingStore>();
/// <summary>
/// Fast options so tests don't stall waiting for the scheduler. 1s busy /
/// 2s idle still exercises the busy-vs-idle branching, but each test
/// completes in &lt; 5 s wall-clock.
/// </summary>
private static IOptions<SiteAuditTelemetryOptions> Opts(
int batchSize = 256,
int busySeconds = 1,
int idleSeconds = 2) =>
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = batchSize,
BusyIntervalSeconds = busySeconds,
IdleIntervalSeconds = idleSeconds,
});
private IActorRef CreateActor(IOptions<SiteAuditTelemetryOptions>? options = null) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
_queue,
_client,
options ?? Opts(),
NullLogger<SiteAuditTelemetryActor>.Instance,
(IOperationTrackingStore?)null)));
/// <summary>
/// AuditLog-001: builds an actor with the optional
/// <see cref="IOperationTrackingStore"/> wired in so the cached-drain
/// scheduler is armed alongside the audit-only drain. Used by the new
/// cached-drain regression tests below.
/// </summary>
private IActorRef CreateActorWithCachedDrain(IOptions<SiteAuditTelemetryOptions>? options = null) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
_queue,
_client,
options ?? Opts(),
NullLogger<SiteAuditTelemetryActor>.Instance,
(IOperationTrackingStore?)_trackingStore)));
private static AuditEvent NewEvent(Guid? id = null) => new()
{
EventId = id ?? Guid.NewGuid(),
OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = "site-1",
ForwardState = AuditForwardState.Pending,
};
private static IngestAck AckAll(IReadOnlyList<AuditEvent> events)
{
var ack = new IngestAck();
foreach (var e in events)
{
ack.AcceptedEventIds.Add(e.EventId.ToString());
}
return ack;
}
[Fact]
public async Task Drain_With_50PendingRows_Sends_OneBatch_Of_50_Then_FlipsToForwarded()
{
// Arrange — 50 pending rows on the first read, then empty on subsequent
// reads so the actor settles after one productive drain.
var pending = Enumerable.Range(0, 50).Select(_ => NewEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(pending),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
AuditEventBatch? capturedBatch = null;
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedBatch = call.Arg<AuditEventBatch>();
return Task.FromResult(AckAll(pending));
});
// Act
CreateActor();
// Assert — give the scheduler time to fire the initial Drain tick.
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestAuditEventsAsync(
Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>());
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 50), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
Assert.NotNull(capturedBatch);
Assert.Equal(50, capturedBatch!.Events.Count);
var expected = pending.Select(e => e.EventId).ToHashSet();
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.ToHashSet().SetEquals(expected)),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task Drain_GrpcThrows_RowsStayPending_NextDrainRetries()
{
// Arrange — first read returns 3 rows; the gRPC client throws on the
// first push, then succeeds on the second. After the second push the
// queue returns empty so the actor settles.
var batch = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(batch),
Task.FromResult<IReadOnlyList<AuditEvent>>(batch),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
var calls = 0;
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(_ =>
{
calls++;
if (calls == 1)
{
throw new InvalidOperationException("simulated gRPC failure");
}
return Task.FromResult(AckAll(batch));
});
// Act
CreateActor();
// Assert — eventually MarkForwardedAsync is called exactly once (after
// the retry succeeded). The first failure must NOT have called
// MarkForwardedAsync because the rows stay Pending.
await AwaitAssertAsync(async () =>
{
await _queue.Received(1).MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(10));
Assert.True(calls >= 2, $"Expected at least 2 client calls (1 failure + 1 retry); saw {calls}");
}
[Fact]
public async Task Drain_ZeroPending_SchedulesAtIdleInterval_NoClientCall()
{
// Arrange — queue always empty.
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
// Idle interval = 2 s. Pause 3 s after the first tick (1 s busy on
// PreStart) and assert the empty-queue branch did NOT push to the
// client.
CreateActor(Opts(busySeconds: 1, idleSeconds: 2));
// Allow the initial tick (~1 s) + a generous window for the idle re-tick.
await Task.Delay(TimeSpan.FromSeconds(3));
await _client.DidNotReceiveWithAnyArgs().IngestAuditEventsAsync(default!, default);
// ReadPendingAsync was called at least once (initial tick), and at
// most twice within the 3 s window (initial + one idle re-tick).
var readCalls = _queue.ReceivedCalls()
.Count(c => c.GetMethodInfo().Name == nameof(ISiteAuditQueue.ReadPendingAsync));
Assert.InRange(readCalls, 1, 2);
}
[Fact]
public async Task Drain_NonZeroPending_SchedulesAtBusyInterval()
{
// Arrange — every read returns 1 row. With busy=1s the actor should
// re-drain quickly, producing multiple client calls inside a short
// window.
var single = new List<AuditEvent> { NewEvent() };
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(single));
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(call => Task.FromResult(AckAll(single)));
CreateActor(Opts(busySeconds: 1, idleSeconds: 10));
// 3-second window with busy=1s should fit at least 2 drains.
await Task.Delay(TimeSpan.FromSeconds(3));
var pushCalls = _client.ReceivedCalls()
.Count(c => c.GetMethodInfo().Name == nameof(ISiteStreamAuditClient.IngestAuditEventsAsync));
Assert.True(pushCalls >= 2,
$"Expected ≥2 pushes within 3s when busy=1s; saw {pushCalls}");
}
[Fact]
public async Task Drain_AcceptedEventIdsSubset_OnlyMarksAccepted()
{
// Arrange — 5 rows pushed, but the central ack only lists 3.
var rows = Enumerable.Range(0, 5).Select(_ => NewEvent()).ToList();
var ackedIds = rows.Take(3).Select(r => r.EventId).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
var partialAck = new IngestAck();
foreach (var id in ackedIds)
{
partialAck.AcceptedEventIds.Add(id.ToString());
}
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult(partialAck));
// Act
CreateActor();
await AwaitAssertAsync(async () =>
{
await _queue.Received(1).MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
// Assert — exactly the 3 ack'd ids made it to MarkForwardedAsync, not
// the other 2.
var ackedSet = ackedIds.ToHashSet();
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 3 && g.ToHashSet().SetEquals(ackedSet)),
Arg.Any<CancellationToken>());
}
// ────────────────────────────────────────────────────────────────────────
// AuditLog-001: combined-telemetry cached-drain regression tests. Verify
// that the production wiring of the previously-unreachable cached transport
// routes cached rows through ReadPendingCachedTelemetryAsync +
// IngestCachedTelemetryAsync (and NOT IngestAuditEventsAsync), and that
// orphaned audit rows (no tracking snapshot) are logged + skipped rather
// than crashing the drain.
// ────────────────────────────────────────────────────────────────────────
private static AuditEvent NewCachedEvent(
AuditKind kind = AuditKind.CachedSubmit,
Guid? eventId = null,
Guid? correlationId = null,
string sourceSiteId = "site-1") => new()
{
EventId = eventId ?? Guid.NewGuid(),
OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
Channel = AuditChannel.ApiOutbound,
Kind = kind,
Status = AuditStatus.Submitted,
SourceSiteId = sourceSiteId,
Target = "ERP.GetOrder",
CorrelationId = correlationId ?? Guid.NewGuid(),
ForwardState = AuditForwardState.Pending,
};
private static TrackingStatusSnapshot NewSnapshot(
TrackedOperationId id,
string status = "Submitted",
int retryCount = 0) => new(
Id: id,
Kind: nameof(AuditKind.ApiCallCached),
TargetSummary: "ERP.GetOrder",
Status: status,
RetryCount: retryCount,
LastError: null,
HttpStatus: null,
CreatedAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
UpdatedAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
TerminalAtUtc: null,
SourceInstanceId: "instance-1",
SourceScript: "script-1",
SourceNode: "node-a");
[Fact]
public async Task CachedDrain_CachedRows_RouteThrough_IngestCachedTelemetry_NotIngestAuditEvents()
{
// Arrange — three cached audit rows on the cached queue, each with a
// matching tracking snapshot. The audit-only queue is empty (those
// rows are excluded by ReadPendingAsync after AuditLog-001).
var cachedRows = new[]
{
NewCachedEvent(AuditKind.CachedSubmit),
NewCachedEvent(AuditKind.ApiCallCached),
NewCachedEvent(AuditKind.CachedResolve),
};
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(cachedRows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
foreach (var row in cachedRows)
{
var tid = new TrackedOperationId(row.CorrelationId!.Value);
_trackingStore.GetStatusAsync(tid, Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(NewSnapshot(tid)));
}
CachedTelemetryBatch? capturedBatch = null;
_client.IngestCachedTelemetryAsync(Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedBatch = call.Arg<CachedTelemetryBatch>();
var ack = new IngestAck();
foreach (var packet in capturedBatch.Packets)
{
ack.AcceptedEventIds.Add(packet.AuditEvent.EventId);
}
return Task.FromResult(ack);
});
// Act
CreateActorWithCachedDrain();
// Assert — exactly one IngestCachedTelemetryAsync push containing all
// three packets, and zero IngestAuditEventsAsync pushes (the audit-only
// drain saw an empty queue).
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestCachedTelemetryAsync(
Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>());
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 3), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
Assert.NotNull(capturedBatch);
Assert.Equal(3, capturedBatch!.Packets.Count);
await _client.DidNotReceiveWithAnyArgs().IngestAuditEventsAsync(default!, default);
var emittedEventIds = capturedBatch.Packets
.Select(p => Guid.Parse(p.AuditEvent.EventId))
.ToHashSet();
var expectedIds = cachedRows.Select(r => r.EventId).ToHashSet();
Assert.Equal(expectedIds, emittedEventIds);
}
[Fact]
public async Task CachedDrain_OrphanRow_NoTrackingSnapshot_IsSkipped_DoesNotCrash()
{
// Arrange — two cached audit rows: one with a tracking snapshot, one
// orphaned (the tracking store returns null). The orphaned row must be
// skipped without aborting the batch — the valid row still flows.
var orphan = NewCachedEvent(AuditKind.CachedSubmit);
var valid = NewCachedEvent(AuditKind.CachedResolve);
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { orphan, valid }),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
// orphan: tracking returns null
_trackingStore.GetStatusAsync(
new TrackedOperationId(orphan.CorrelationId!.Value),
Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(null));
// valid: tracking returns a snapshot
var validTid = new TrackedOperationId(valid.CorrelationId!.Value);
_trackingStore.GetStatusAsync(validTid, Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(NewSnapshot(validTid, "Delivered")));
CachedTelemetryBatch? capturedBatch = null;
_client.IngestCachedTelemetryAsync(Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedBatch = call.Arg<CachedTelemetryBatch>();
var ack = new IngestAck();
foreach (var packet in capturedBatch.Packets)
{
ack.AcceptedEventIds.Add(packet.AuditEvent.EventId);
}
return Task.FromResult(ack);
});
// Act
CreateActorWithCachedDrain();
// Assert — exactly one push containing ONLY the valid row; the orphan
// is skipped and stays Pending (not in MarkForwardedAsync's id list).
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestCachedTelemetryAsync(
Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
Assert.NotNull(capturedBatch);
Assert.Single(capturedBatch!.Packets);
Assert.Equal(valid.EventId.ToString(), capturedBatch.Packets[0].AuditEvent.EventId);
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == valid.EventId),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task AuditOnlyDrain_StillFlows_When_CachedDrain_IsDisabled()
{
// Arrange — ordinary (non-cached) audit rows on the audit-only queue;
// the actor is constructed WITHOUT a tracking store so the cached
// scheduler is never armed. Regression guard against the audit-only
// drain regressing during the AuditLog-001 refactor.
var rows = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(_ => Task.FromResult(AckAll(rows)));
// Act — note: CreateActor (no tracking store), not CreateActorWithCachedDrain.
CreateActor();
// Assert — audit-only drain flows normally; the cached client is
// never called and ReadPendingCachedTelemetryAsync is never queried.
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestAuditEventsAsync(
Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>());
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 3), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
await _client.DidNotReceiveWithAnyArgs().IngestCachedTelemetryAsync(default!, default);
await _queue.DidNotReceiveWithAnyArgs().ReadPendingCachedTelemetryAsync(default, default);
}
}