refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)

Solution + 23 src projects + 26 test projects renamed; folders, csproj,
namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated.
ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated.
SQL roles/logins, LDAP domains, CLI command name, and CLI config dir
(~/.scadalink → ~/.scadabridge) also renamed.

Build green; 5 Host.Tests fail awaiting SQL login rename in next commit.
Pre-existing StaleTagMonitor timing flakes unchanged.

Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
Joseph Doherty
2026-05-28 09:37:45 -04:00
parent 6d87ee3c3b
commit 7b0b9c7365
1531 changed files with 11180 additions and 11054 deletions
@@ -0,0 +1,477 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using System.Net;
using System.Security.Claims;
using System.Text;
using System.Text.Encodings.Web;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Audit Log #23 — M4 Bundle E (Task E4) cross-boundary safety suite verifying
/// the alog.md §13 contract: an always-throwing audit writer NEVER aborts the
/// user-facing action. Exercises every boundary that emits audit rows in M2,
/// M3, and M4:
///
/// <list type="bullet">
/// <item><description>External system sync call (M2 Bundle F).</description></item>
/// <item><description>External system cached call (M3 Bundle E).</description></item>
/// <item><description>Database sync write (M4 Bundle A).</description></item>
/// <item><description>Inbound API request (M4 Bundle D).</description></item>
/// <item><description>Notification dispatcher (M4 Bundle B).</description></item>
/// </list>
///
/// <para>
/// The site-local boundaries (ESG sync/cached, DB sync) take the always-throw
/// <see cref="ThrowingAuditWriter"/> in place of the production
/// <see cref="IAuditWriter"/>; the central boundaries (Inbound API,
/// Notification dispatcher) take the always-throw
/// <see cref="ThrowingCentralAuditWriter"/> in place of
/// <see cref="ICentralAuditWriter"/>. In each case the wrapped action's
/// original return value (or original exception) must still flow back to the
/// caller untouched.
/// </para>
/// </remarks>
public class AuditWriteFailureSafetyTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public AuditWriteFailureSafetyTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
// ---------------------------------------------------------------------
// Always-throwing writer test doubles
// ---------------------------------------------------------------------
/// <summary>
/// Site-side <see cref="IAuditWriter"/> that ALWAYS throws on
/// <see cref="WriteAsync"/>. Used to verify that ESG / DB script-side
/// helpers swallow the throw and return their normal result to the script.
/// </summary>
private sealed class ThrowingAuditWriter : IAuditWriter
{
private int _attempts;
public int Attempts => Volatile.Read(ref _attempts);
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
{
Interlocked.Increment(ref _attempts);
return Task.FromException(new InvalidOperationException(
"test-only ThrowingAuditWriter — audit pipeline unavailable"));
}
}
/// <summary>
/// Central-side <see cref="ICentralAuditWriter"/> that ALWAYS throws on
/// <see cref="WriteAsync"/>. Used to verify Inbound API + Notification
/// dispatcher absorb audit-write failures rather than propagating them
/// into the response / state transition.
/// </summary>
private sealed class ThrowingCentralAuditWriter : ICentralAuditWriter
{
private int _attempts;
public int Attempts => Volatile.Read(ref _attempts);
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
{
Interlocked.Increment(ref _attempts);
throw new InvalidOperationException(
"test-only ThrowingCentralAuditWriter — audit subsystem unavailable");
}
}
/// <summary>
/// Site-side <see cref="ICachedCallTelemetryForwarder"/> that ALWAYS
/// throws on <see cref="ForwardAsync"/>. The cached-call helpers absorb
/// the throw and still return a valid <see cref="TrackedOperationId"/>.
/// </summary>
private sealed class ThrowingCachedForwarder : ICachedCallTelemetryForwarder
{
private int _attempts;
public int Attempts => Volatile.Read(ref _attempts);
public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default)
{
Interlocked.Increment(ref _attempts);
return Task.FromException(new InvalidOperationException(
"test-only ThrowingCachedForwarder — telemetry pipeline unavailable"));
}
}
// ---------------------------------------------------------------------
// Test 1 — ESG sync call still returns the original ExternalCallResult.
// ---------------------------------------------------------------------
[Fact]
public async Task EsgSyncCall_BrokenAuditWriter_StillReturnsResult()
{
var client = Substitute.For<IExternalSystemClient>();
var expected = new ExternalCallResult(
Success: true,
ResponseJson: "{\"orderId\":42}",
ErrorMessage: null,
WasBuffered: false);
client.CallAsync(
"ERP", "GetOrder",
Arg.Any<IReadOnlyDictionary<string, object?>?>(),
Arg.Any<CancellationToken>())
.Returns(expected);
var writer = new ThrowingAuditWriter();
var helper = new ScriptRuntimeContext.ExternalSystemHelper(
client,
instanceName: "Plant.Pump42",
NullLogger.Instance,
Guid.NewGuid(),
auditWriter: writer,
siteId: "site-77",
sourceScript: "ScriptActor:Sync",
cachedForwarder: null);
var result = await helper.Call("ERP", "GetOrder");
Assert.Same(expected, result);
// Proof the audit writer was attempted — otherwise the test wouldn't
// actually exercise the safety contract.
Assert.True(writer.Attempts >= 1,
$"Expected audit writer to be invoked at least once; saw {writer.Attempts}.");
}
// ---------------------------------------------------------------------
// Test 2 — ESG cached call still returns a TrackedOperationId.
// ---------------------------------------------------------------------
[Fact]
public async Task EsgCachedCall_BrokenAuditWriter_StillReturnsTrackedOperationId()
{
var client = Substitute.For<IExternalSystemClient>();
// CachedCallAsync returns WasBuffered=true so the helper takes the
// S&F-deferred path — no immediate-terminal telemetry, which keeps the
// forwarder attempt count at exactly one (the CachedSubmit emission).
client.CachedCallAsync(
"ERP", "GetOrder",
Arg.Any<IReadOnlyDictionary<string, object?>?>(),
Arg.Any<string?>(),
Arg.Any<CancellationToken>(),
Arg.Any<TrackedOperationId?>())
.Returns(new ExternalCallResult(true, null, null, WasBuffered: true));
// BOTH the audit writer AND the cached forwarder throw — the
// CachedSubmit emission goes through the forwarder in production, so
// breaking only the writer wouldn't actually exercise the cached
// path's safety contract.
var writer = new ThrowingAuditWriter();
var forwarder = new ThrowingCachedForwarder();
var helper = new ScriptRuntimeContext.ExternalSystemHelper(
client,
instanceName: "Plant.Pump42",
NullLogger.Instance,
Guid.NewGuid(),
auditWriter: writer,
siteId: "site-77",
sourceScript: "ScriptActor:Cached",
cachedForwarder: forwarder);
var trackedId = await helper.CachedCall("ERP", "GetOrder");
// Non-default id materialised despite the forwarder failing.
Assert.NotEqual(default, trackedId);
Assert.True(forwarder.Attempts >= 1,
$"Expected cached forwarder to be invoked at least once; saw {forwarder.Attempts}.");
}
// ---------------------------------------------------------------------
// Test 3 — DB sync write still returns the rows-affected count.
// ---------------------------------------------------------------------
[Fact]
public async Task DbSyncWrite_BrokenAuditWriter_StillReturnsRowsAffected()
{
const string connectionName = "machineData";
const string instanceName = "Plant.Pump42";
using var keepAlive = new SqliteConnection(
"Data Source=k-safety-db;Mode=Memory;Cache=Shared");
keepAlive.Open();
// Schema + seed inside a unique in-memory DB.
var dbName = $"db-{Guid.NewGuid():N}";
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
using var dbKeepAlive = new SqliteConnection(connStr);
dbKeepAlive.Open();
using (var seed = dbKeepAlive.CreateCommand())
{
seed.CommandText =
"CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);";
seed.ExecuteNonQuery();
}
var inner = new SqliteConnection(connStr);
inner.Open();
var gateway = Substitute.For<IDatabaseGateway>();
gateway.GetConnectionAsync(connectionName, Arg.Any<CancellationToken>())
.Returns(inner);
var writer = new ThrowingAuditWriter();
var helper = new ScriptRuntimeContext.DatabaseHelper(
gateway,
instanceName,
NullLogger.Instance,
Guid.NewGuid(),
auditWriter: writer,
siteId: "site-77",
sourceScript: "ScriptActor:Db",
cachedForwarder: null);
await using (var conn = await helper.Connection(connectionName))
await using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "INSERT INTO t (id, name) VALUES (1, 'safety')";
var rows = await cmd.ExecuteNonQueryAsync();
Assert.Equal(1, rows);
}
Assert.True(writer.Attempts >= 1,
$"Expected audit writer to be invoked at least once; saw {writer.Attempts}.");
}
// ---------------------------------------------------------------------
// Test 4 — Inbound API request still returns HTTP 200.
// ---------------------------------------------------------------------
[Fact]
public async Task InboundApi_BrokenAuditWriter_StillReturns200()
{
var writer = new ThrowingCentralAuditWriter();
using var host = await BuildInboundApiHostAsync(writer, endpointStatus: 200);
var client = host.GetTestClient();
var resp = await client.PostAsync(
"/api/echo",
new StringContent("{\"x\":1}", Encoding.UTF8, "application/json"));
Assert.Equal(HttpStatusCode.OK, resp.StatusCode);
Assert.True(writer.Attempts >= 1,
$"Expected central audit writer to be invoked at least once; saw {writer.Attempts}.");
}
// ---------------------------------------------------------------------
// Test 5 — Notification dispatcher still transitions to Delivered.
// ---------------------------------------------------------------------
[SkippableFact]
public async Task NotificationDispatch_BrokenAuditWriter_StillTransitionsToDelivered()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = "test-e4-safety-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var notificationId = Guid.NewGuid();
await SeedSmtpConfigAsync();
await SeedNotificationAsync(notificationId, siteId);
var adapter = new SingleOutcomeAdapter(DeliveryOutcome.Success("ops@example.com"));
var serviceProvider = BuildNotificationDispatcherProvider(adapter);
var throwingWriter = new ThrowingCentralAuditWriter();
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
serviceProvider,
new NotificationOutboxOptions
{
DispatchInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromDays(1),
},
(ICentralAuditWriter)throwingWriter,
NullLogger<NotificationOutboxActor>.Instance)));
actor.Tell(InternalMessages.DispatchTick.Instance);
// Notifications table reflects the successful delivery even though
// every audit write threw — the central direct-write writer
// catches/logs internally and the dispatcher catches defensively too
// (alog.md §13).
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var row = await ctx.Notifications.SingleAsync(
n => n.NotificationId == notificationId.ToString("D"));
Assert.Equal(NotificationStatus.Delivered, row.Status);
Assert.NotNull(row.DeliveredAt);
}, TimeSpan.FromSeconds(15));
Assert.True(throwingWriter.Attempts >= 1,
$"Expected dispatcher to attempt audit write at least once; saw {throwingWriter.Attempts}.");
}
// ---------------------------------------------------------------------
// Test infrastructure
// ---------------------------------------------------------------------
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
.Options;
return new ScadaBridgeDbContext(options);
}
private IServiceProvider BuildNotificationDispatcherProvider(
INotificationDeliveryAdapter adapter)
{
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(opts =>
opts.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
services.AddScoped<INotificationOutboxRepository>(sp =>
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
services.AddScoped<INotificationRepository>(sp =>
new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
return services.BuildServiceProvider();
}
private async Task SeedSmtpConfigAsync()
{
await using var ctx = CreateContext();
// NO-002: dispatcher clamps non-positive RetryDelay to the 1-minute fallback;
// use 1 ms so a transient outcome's NextAttemptAt is still effectively due.
ctx.SmtpConfigurations.Add(new SmtpConfiguration(
"smtp.example.com", "Basic", "noreply@example.com")
{
MaxRetries = 5,
RetryDelay = TimeSpan.FromMilliseconds(1),
});
await ctx.SaveChangesAsync();
}
private async Task SeedNotificationAsync(Guid notificationId, string siteId)
{
await using var ctx = CreateContext();
ctx.Notifications.Add(new Notification(
notificationId.ToString("D"),
NotificationType.Email,
"ops-team",
"Safety subject",
"Safety body",
siteId)
{
SourceInstanceId = "Plant.Pump42",
SourceScript = "AlarmScript",
CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1),
});
await ctx.SaveChangesAsync();
}
/// <summary>
/// Single-outcome adapter — returns the same <see cref="DeliveryOutcome"/>
/// for every call. Used by the dispatcher safety test where we only need
/// one happy-path delivery.
/// </summary>
private sealed class SingleOutcomeAdapter : INotificationDeliveryAdapter
{
private readonly DeliveryOutcome _outcome;
public SingleOutcomeAdapter(DeliveryOutcome outcome) { _outcome = outcome; }
public NotificationType Type => NotificationType.Email;
public Task<DeliveryOutcome> DeliverAsync(
Notification notification, CancellationToken cancellationToken = default)
=> Task.FromResult(_outcome);
}
/// <summary>
/// Builds an in-memory TestHost mirroring the production inbound-API
/// pipeline order. The supplied <paramref name="writer"/> stands in for
/// the production <see cref="ICentralAuditWriter"/> so the safety test can
/// install the always-throwing variant without standing up any DB.
/// </summary>
private static async Task<IHost> BuildInboundApiHostAsync(
ICentralAuditWriter writer, int endpointStatus)
{
var hostBuilder = new HostBuilder()
.ConfigureWebHost(webBuilder =>
{
webBuilder
.UseTestServer()
.ConfigureServices(services =>
{
services.AddSingleton(writer);
services.AddRouting();
services.AddAuthorization();
services.AddAuthentication("TestScheme")
.AddScheme<Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions,
AlwaysAuthenticatedHandler>("TestScheme", _ => { });
})
.Configure(app =>
{
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseAuditWriteMiddleware();
app.UseEndpoints(endpoints =>
{
endpoints.MapPost("/api/{methodName}", async ctx =>
{
ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "safety-actor";
ctx.Response.StatusCode = endpointStatus;
await ctx.Response.WriteAsync("ok");
});
});
});
});
return await hostBuilder.StartAsync();
}
private sealed class AlwaysAuthenticatedHandler
: Microsoft.AspNetCore.Authentication.AuthenticationHandler<
Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions>
{
public AlwaysAuthenticatedHandler(
IOptionsMonitor<Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions> options,
Microsoft.Extensions.Logging.ILoggerFactory logger,
UrlEncoder encoder)
: base(options, logger, encoder) { }
protected override Task<Microsoft.AspNetCore.Authentication.AuthenticateResult>
HandleAuthenticateAsync()
{
var identity = new ClaimsIdentity(
new[] { new Claim(ClaimTypes.Name, "framework-user") }, "TestScheme");
var principal = new ClaimsPrincipal(identity);
var ticket = new Microsoft.AspNetCore.Authentication.AuthenticationTicket(
principal, "TestScheme");
return Task.FromResult(
Microsoft.AspNetCore.Authentication.AuthenticateResult.Success(ticket));
}
}
}
@@ -0,0 +1,271 @@
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle G G2 end-to-end suite for cached <c>ExternalSystem.CachedCall</c>
/// lifecycle telemetry (Audit Log #23 / M3). Wires the full M3 pipeline:
/// site-local SQLite audit writer + operation tracking store + the production
/// <see cref="CachedCallTelemetryForwarder"/> + the test-side
/// <see cref="CombinedTelemetryDispatcher"/> that ALSO pushes each combined
/// packet through the stub gRPC client into the central
/// <c>AuditLogIngestActor</c>'s dual-write transaction against a per-test
/// MSSQL database. Asserts the audit rows + the SiteCalls row + the
/// site-local tracking row converge to the expected shape for each lifecycle.
/// </summary>
/// <remarks>
/// <para>
/// The bridge is driven directly via <see cref="CombinedTelemetryHarness.EmitAttemptAsync"/>
/// — these tests do NOT spin up the actual S&amp;F retry loop; that would
/// require a full SiteRuntime host and is out of scope for M3 (the S&amp;F
/// observer hooks are exercised in <c>ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests</c> at
/// unit level). The submit row is emitted via
/// <see cref="CombinedTelemetryHarness.EmitSubmitAsync"/> because the
/// production submit emission happens at the script-call site, not inside the
/// S&amp;F loop.
/// </para>
/// <para>
/// Each test uses a unique <c>SourceSite</c> id (Guid suffix) so concurrent
/// tests sharing the per-fixture MSSQL database don't interfere with each
/// other.
/// </para>
/// </remarks>
public class CachedCallCombinedTelemetryTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public CachedCallCombinedTelemetryTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private static string NewSiteId() =>
"test-g2-cached-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private static CachedCallTelemetry SubmitPacket(
TrackedOperationId id, string siteId, DateTime nowUtc, string target = "ERP.GetOrder") =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = nowUtc,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.CachedSubmit,
CorrelationId = id.Value,
SourceSiteId = siteId,
SourceInstanceId = "Plant.Pump42",
SourceScript = "ScriptActor:doStuff",
Target = target,
Status = AuditStatus.Submitted,
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: id,
Channel: "ApiOutbound",
Target: target,
SourceSite: siteId,
SourceNode: null,
Status: "Submitted",
RetryCount: 0,
LastError: null,
HttpStatus: null,
CreatedAtUtc: nowUtc,
UpdatedAtUtc: nowUtc,
TerminalAtUtc: null));
private static CachedCallAttemptContext AttemptContext(
TrackedOperationId id,
string siteId,
CachedCallAttemptOutcome outcome,
int retryCount,
string? lastError,
int? httpStatus,
DateTime createdUtc,
DateTime occurredUtc,
string target = "ERP.GetOrder",
string channel = "ApiOutbound") =>
new(
TrackedOperationId: id,
Channel: channel,
Target: target,
SourceSite: siteId,
Outcome: outcome,
RetryCount: retryCount,
LastError: lastError,
HttpStatus: httpStatus,
CreatedAtUtc: createdUtc,
OccurredAtUtc: occurredUtc,
DurationMs: 42,
SourceInstanceId: "Plant.Pump42");
[SkippableFact]
public async Task CachedCall_FailFailSuccess_Emits_5_AuditRows_AND_1_SiteCall_Delivered()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var t0 = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
// Submit
await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0));
// Attempt 1: transient HTTP 500
await harness.EmitAttemptAsync(AttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.TransientFailure,
retryCount: 1, lastError: "HTTP 500", httpStatus: 500,
createdUtc: t0, occurredUtc: t0.AddSeconds(5)));
// Attempt 2: transient HTTP 500
await harness.EmitAttemptAsync(AttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.TransientFailure,
retryCount: 2, lastError: "HTTP 500", httpStatus: 500,
createdUtc: t0, occurredUtc: t0.AddSeconds(15)));
// Attempt 3: delivered (terminal — emits Attempted + CachedResolve)
await harness.EmitAttemptAsync(AttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.Delivered,
retryCount: 3, lastError: null, httpStatus: 200,
createdUtc: t0, occurredUtc: t0.AddSeconds(25)));
// Central side: each forward through the dispatcher round-trips
// through the stub client + ingest actor, so by the time the awaits
// complete the rows are visible in MSSQL.
await using var read = harness.CreateReadContext();
// 1 Submit + 2 transient Attempted + 1 terminal Attempted + 1
// CachedResolve = 5 audit rows. The plan allows 4-5; this is the
// happy path emitting exactly 5.
var auditRows = await read.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.InRange(auditRows.Count, 4, 5);
// All audit rows must share the same CorrelationId (= TrackedOperationId).
Assert.All(auditRows, r => Assert.Equal(trackedId.Value, r.CorrelationId));
// Exactly one CachedSubmit row.
Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit);
// Exactly one terminal CachedResolve row, status Delivered.
var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve);
Assert.Equal(AuditStatus.Delivered, resolve.Status);
// SiteCalls row: Delivered, RetryCount=3, TerminalAtUtc set.
var siteCall = await read.Set<SiteCall>()
.SingleAsync(s => s.TrackedOperationId == trackedId);
Assert.Equal("Delivered", siteCall.Status);
Assert.Equal(3, siteCall.RetryCount);
Assert.NotNull(siteCall.TerminalAtUtc);
// Site-local Tracking.Status mirrors the same outcome.
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
Assert.NotNull(snapshot);
Assert.Equal("Delivered", snapshot!.Status);
Assert.NotNull(snapshot.TerminalAtUtc);
}
[SkippableFact]
public async Task CachedCall_AllAttemptsFailedAndParked_Emits_Terminal_Parked()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var t0 = new DateTime(2026, 5, 20, 11, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0));
// Three transient failures...
for (int i = 1; i <= 3; i++)
{
await harness.EmitAttemptAsync(AttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.TransientFailure,
retryCount: i, lastError: "HTTP 500", httpStatus: 500,
createdUtc: t0, occurredUtc: t0.AddSeconds(i * 5)));
}
// ...then S&F gives up — ParkedMaxRetries.
await harness.EmitAttemptAsync(AttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.ParkedMaxRetries,
retryCount: 4, lastError: "HTTP 500", httpStatus: 500,
createdUtc: t0, occurredUtc: t0.AddSeconds(30)));
await using var read = harness.CreateReadContext();
var siteCall = await read.Set<SiteCall>()
.SingleAsync(s => s.TrackedOperationId == trackedId);
Assert.Equal("Parked", siteCall.Status);
Assert.NotNull(siteCall.TerminalAtUtc);
// Terminal audit row should also be Parked.
var resolve = await read.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId && e.Kind == AuditKind.CachedResolve)
.SingleAsync();
Assert.Equal(AuditStatus.Parked, resolve.Status);
// Site-local tracking matches.
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
Assert.NotNull(snapshot);
Assert.Equal("Parked", snapshot!.Status);
Assert.NotNull(snapshot.TerminalAtUtc);
}
[SkippableFact]
public async Task CachedCall_ImmediateSuccess_NoSF_Emits_Attempted_And_Resolve_Delivered()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
// Submit + immediate delivered attempt (RetryCount = 0).
await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0));
await harness.EmitAttemptAsync(AttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.Delivered,
retryCount: 0, lastError: null, httpStatus: 200,
createdUtc: t0, occurredUtc: t0.AddMilliseconds(50)));
await using var read = harness.CreateReadContext();
var siteCall = await read.Set<SiteCall>()
.SingleAsync(s => s.TrackedOperationId == trackedId);
Assert.Equal("Delivered", siteCall.Status);
Assert.Equal(0, siteCall.RetryCount);
Assert.NotNull(siteCall.TerminalAtUtc);
// 1 Submit + 1 Attempted + 1 CachedResolve = 3 audit rows.
var auditRows = await read.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(3, auditRows.Count);
Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit);
Assert.Single(auditRows, r => r.Kind == AuditKind.ApiCallCached);
var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve);
Assert.Equal(AuditStatus.Delivered, resolve.Status);
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
Assert.NotNull(snapshot);
Assert.Equal("Delivered", snapshot!.Status);
}
}
@@ -0,0 +1,197 @@
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle G G3 mirror of <see cref="CachedCallCombinedTelemetryTests"/> for
/// <c>Database.CachedWrite</c>. Same pipeline composition, same dual-write
/// transaction, but the lifecycle bridge maps channel <c>"DbOutbound"</c> to
/// <see cref="AuditKind.DbWriteCached"/> on the per-attempt audit row (vs.
/// <see cref="AuditKind.ApiCallCached"/> for API calls). The
/// <see cref="AuditChannel"/> on the audit row, the <c>SiteCalls.Channel</c>
/// column, and the per-attempt <see cref="AuditKind"/> all need to come
/// through as the DB variants for this path to be considered exercised.
/// </summary>
/// <remarks>
/// As with G2, the bridge is driven directly via the harness — we do not
/// stand up a real <c>Database.CachedWrite</c> caller. The site-side
/// unit-level emission for the DB path is exercised in
/// <c>ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests</c>; this suite verifies the end-to-end
/// combined-telemetry path produces the right central rows.
/// </remarks>
public class CachedWriteCombinedTelemetryTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public CachedWriteCombinedTelemetryTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private static string NewSiteId() =>
"test-g3-cachedwrite-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private static CachedCallTelemetry DbSubmitPacket(
TrackedOperationId id, string siteId, DateTime nowUtc, string target = "OperationsDb.UpdateOrder") =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = nowUtc,
Channel = AuditChannel.DbOutbound,
Kind = AuditKind.CachedSubmit,
CorrelationId = id.Value,
SourceSiteId = siteId,
SourceInstanceId = "Plant.Pump42",
SourceScript = "ScriptActor:doStuff",
Target = target,
Status = AuditStatus.Submitted,
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: id,
Channel: "DbOutbound",
Target: target,
SourceSite: siteId,
SourceNode: null,
Status: "Submitted",
RetryCount: 0,
LastError: null,
HttpStatus: null,
CreatedAtUtc: nowUtc,
UpdatedAtUtc: nowUtc,
TerminalAtUtc: null));
private static CachedCallAttemptContext DbAttemptContext(
TrackedOperationId id,
string siteId,
CachedCallAttemptOutcome outcome,
int retryCount,
string? lastError,
DateTime createdUtc,
DateTime occurredUtc,
string target = "OperationsDb.UpdateOrder") =>
new(
TrackedOperationId: id,
Channel: "DbOutbound",
Target: target,
SourceSite: siteId,
Outcome: outcome,
RetryCount: retryCount,
LastError: lastError,
HttpStatus: null,
CreatedAtUtc: createdUtc,
OccurredAtUtc: occurredUtc,
DurationMs: 12,
SourceInstanceId: "Plant.Pump42");
[SkippableFact]
public async Task CachedWrite_Success_Emits_Delivered_Lifecycle()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var t0 = new DateTime(2026, 5, 20, 13, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
// Submit + immediate delivered attempt.
await harness.EmitSubmitAsync(DbSubmitPacket(trackedId, siteId, t0));
await harness.EmitAttemptAsync(DbAttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.Delivered,
retryCount: 0, lastError: null,
createdUtc: t0, occurredUtc: t0.AddMilliseconds(50)));
await using var read = harness.CreateReadContext();
// Central SiteCalls row — DbOutbound channel, Delivered.
var siteCall = await read.Set<SiteCall>()
.SingleAsync(s => s.TrackedOperationId == trackedId);
Assert.Equal("DbOutbound", siteCall.Channel);
Assert.Equal("Delivered", siteCall.Status);
Assert.Equal(0, siteCall.RetryCount);
Assert.NotNull(siteCall.TerminalAtUtc);
var auditRows = await read.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(3, auditRows.Count);
// Submit row: CachedSubmit + DbOutbound channel.
var submit = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit);
Assert.Equal(AuditChannel.DbOutbound, submit.Channel);
// Per-attempt row: DbWriteCached (NOT ApiCallCached).
var attempt = Assert.Single(auditRows, r => r.Kind == AuditKind.DbWriteCached);
Assert.Equal(AuditStatus.Attempted, attempt.Status);
Assert.Equal(AuditChannel.DbOutbound, attempt.Channel);
// Terminal: CachedResolve Delivered.
var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve);
Assert.Equal(AuditStatus.Delivered, resolve.Status);
Assert.Equal(AuditChannel.DbOutbound, resolve.Channel);
// Site-local tracking row mirrors the same outcome.
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
Assert.NotNull(snapshot);
Assert.Equal("Delivered", snapshot!.Status);
}
[SkippableFact]
public async Task CachedWrite_Parked_Emits_Terminal_Parked()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var t0 = new DateTime(2026, 5, 20, 14, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
await harness.EmitSubmitAsync(DbSubmitPacket(trackedId, siteId, t0));
// Two transient SQL-error attempts...
for (int i = 1; i <= 2; i++)
{
await harness.EmitAttemptAsync(DbAttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.TransientFailure,
retryCount: i, lastError: "Deadlock victim",
createdUtc: t0, occurredUtc: t0.AddSeconds(i * 5)));
}
// ...then permanent failure → Parked terminal.
await harness.EmitAttemptAsync(DbAttemptContext(
trackedId, siteId,
CachedCallAttemptOutcome.PermanentFailure,
retryCount: 3, lastError: "ConstraintViolation: FK_Orders_Customer",
createdUtc: t0, occurredUtc: t0.AddSeconds(20)));
await using var read = harness.CreateReadContext();
var siteCall = await read.Set<SiteCall>()
.SingleAsync(s => s.TrackedOperationId == trackedId);
Assert.Equal("DbOutbound", siteCall.Channel);
Assert.Equal("Parked", siteCall.Status);
Assert.NotNull(siteCall.TerminalAtUtc);
var resolve = await read.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId && e.Kind == AuditKind.CachedResolve)
.SingleAsync();
Assert.Equal(AuditStatus.Parked, resolve.Status);
Assert.Equal(AuditChannel.DbOutbound, resolve.Channel);
// Tracking store mirrors Parked.
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
Assert.NotNull(snapshot);
Assert.Equal("Parked", snapshot!.Status);
Assert.NotNull(snapshot.TerminalAtUtc);
}
}
@@ -0,0 +1,201 @@
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
using Timestamp = Google.Protobuf.WellKnownTypes.Timestamp;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle G G4 idempotency suite. Telemetry packets MUST round-trip safely
/// under retried delivery (at-least-once site→central) AND under out-of-order
/// arrival (a stale Submit packet arriving after the central row has already
/// advanced to Attempted must not regress the SiteCalls status, but must
/// still insert its own audit row because audit rows are append-only and the
/// lifecycle history is the source of truth for forensics).
/// </summary>
/// <remarks>
/// Pushes <see cref="CachedTelemetryBatch"/> packets directly through the
/// stub client (bypassing the local SQLite writer + tracking store) — the
/// scenario being modeled is a wire-level retry, not a fresh site call, so
/// the local stores' insert/no-op behaviour is already covered by the G2/G3
/// happy-path tests. This suite focuses on the central ingest actor's
/// dual-write transaction's idempotency contract.
/// </remarks>
public class CombinedTelemetryIdempotencyTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public CombinedTelemetryIdempotencyTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private static string NewSiteId() =>
"test-g4-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private static CachedTelemetryPacket BuildPacket(
Guid eventId,
TrackedOperationId trackedId,
string siteId,
AuditKind kind,
AuditStatus auditStatus,
string operationalStatus,
int retryCount,
DateTime nowUtc,
DateTime? terminalUtc = null,
string? lastError = null,
int? httpStatus = null)
{
var dto = new CachedTelemetryPacket
{
AuditEvent = AuditEventDtoMapper.ToDto(new AuditEvent
{
EventId = eventId,
OccurredAtUtc = nowUtc,
Channel = AuditChannel.ApiOutbound,
Kind = kind,
CorrelationId = trackedId.Value,
SourceSiteId = siteId,
Target = "ERP.GetOrder",
Status = auditStatus,
HttpStatus = httpStatus,
ErrorMessage = lastError,
ForwardState = AuditForwardState.Pending,
}),
Operational = new SiteCallOperationalDto
{
TrackedOperationId = trackedId.Value.ToString("D"),
Channel = "ApiOutbound",
Target = "ERP.GetOrder",
SourceSite = siteId,
Status = operationalStatus,
RetryCount = retryCount,
LastError = lastError ?? string.Empty,
CreatedAtUtc = Timestamp.FromDateTime(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc)),
UpdatedAtUtc = Timestamp.FromDateTime(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc)),
},
};
if (httpStatus.HasValue)
{
dto.Operational.HttpStatus = httpStatus.Value;
}
if (terminalUtc.HasValue)
{
dto.Operational.TerminalAtUtc =
Timestamp.FromDateTime(DateTime.SpecifyKind(terminalUtc.Value, DateTimeKind.Utc));
}
return dto;
}
private static CachedTelemetryBatch BatchOf(params CachedTelemetryPacket[] packets)
{
var batch = new CachedTelemetryBatch();
batch.Packets.AddRange(packets);
return batch;
}
[SkippableFact]
public async Task DuplicatePacket_AuditLogStaysAtOneRow_SiteCallsUpserted_Monotonically()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var eventId = Guid.NewGuid();
var t0 = new DateTime(2026, 5, 20, 15, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
var packet = BuildPacket(
eventId, trackedId, siteId,
AuditKind.CachedSubmit, AuditStatus.Submitted, "Submitted",
retryCount: 0, nowUtc: t0);
// First delivery
var ack1 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(packet), CancellationToken.None);
Assert.Single(ack1.AcceptedEventIds);
// Second delivery — the exact same packet (simulates a retried gRPC).
var ack2 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(packet), CancellationToken.None);
// Central acks both deliveries because storage state is consistent —
// the site is free to treat its local row as Forwarded either way.
Assert.Single(ack2.AcceptedEventIds);
Assert.Equal(eventId.ToString(), ack2.AcceptedEventIds[0]);
await using var read = harness.CreateReadContext();
// AuditLog: exactly ONE row for the EventId (insert-if-not-exists).
var auditCount = await read.Set<AuditEvent>()
.CountAsync(e => e.EventId == eventId);
Assert.Equal(1, auditCount);
// SiteCalls: exactly ONE row for the TrackedOperationId.
var siteCalls = await read.Set<SiteCall>()
.Where(s => s.TrackedOperationId == trackedId)
.ToListAsync();
Assert.Single(siteCalls);
Assert.Equal("Submitted", siteCalls[0].Status);
Assert.Equal(0, siteCalls[0].RetryCount);
}
[SkippableFact]
public async Task OutOfOrderPackets_OlderStatus_ArrivesAfterNewer_IsNoOp()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var trackedId = TrackedOperationId.New();
var t0 = new DateTime(2026, 5, 20, 16, 0, 0, DateTimeKind.Utc);
await using var harness = new CombinedTelemetryHarness(_fixture, this);
// First: the Attempted (RetryCount=2) row arrives at central — perhaps
// the Submit packet got delayed in flight. SiteCalls advances straight
// to Attempted with retry count 2.
var attemptedEventId = Guid.NewGuid();
var attemptedPacket = BuildPacket(
attemptedEventId, trackedId, siteId,
AuditKind.ApiCallCached, AuditStatus.Attempted, "Attempted",
retryCount: 2, nowUtc: t0.AddSeconds(10),
lastError: "HTTP 500", httpStatus: 500);
var ack1 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(attemptedPacket), CancellationToken.None);
Assert.Single(ack1.AcceptedEventIds);
// Now the stale Submit packet shows up. The audit row should still be
// inserted (audit is append-only — preserve the lifecycle history),
// but SiteCalls must NOT regress to Submitted/RetryCount=0.
var submitEventId = Guid.NewGuid();
var submitPacket = BuildPacket(
submitEventId, trackedId, siteId,
AuditKind.CachedSubmit, AuditStatus.Submitted, "Submitted",
retryCount: 0, nowUtc: t0);
var ack2 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(submitPacket), CancellationToken.None);
Assert.Single(ack2.AcceptedEventIds);
await using var read = harness.CreateReadContext();
// AuditLog: TWO rows now exist for this lifecycle — the Submit and
// the Attempted. Their order is by OccurredAtUtc; the test doesn't
// assert ordering, only count + correlation.
var auditRows = await read.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(2, auditRows.Count);
Assert.All(auditRows, r => Assert.Equal(trackedId.Value, r.CorrelationId));
// SiteCalls: stuck at Attempted (monotonic — Submitted is rank 0,
// Attempted is rank 2, the upsert for the older row is a no-op).
var siteCall = await read.Set<SiteCall>()
.SingleAsync(s => s.TrackedOperationId == trackedId);
Assert.Equal("Attempted", siteCall.Status);
Assert.Equal(2, siteCall.RetryCount);
Assert.Equal("HTTP 500", siteCall.LastError);
Assert.Equal(500, siteCall.HttpStatus);
}
}
@@ -0,0 +1,300 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Audit Log #23 — M4 Bundle E (Task E1) end-to-end suite verifying every
/// synchronous <c>Database.Connection(name).Execute*</c> /
/// <c>ExecuteReader</c> call made via the Bundle A
/// <see cref="ScriptRuntimeContext.DatabaseHelper"/> emits exactly one
/// <see cref="AuditChannel.DbOutbound"/>/<see cref="AuditKind.DbWrite"/> row
/// that materialises in the central MSSQL <c>AuditLog</c> via the production
/// site-SQLite + telemetry-actor + central ingest-actor pipeline.
/// </summary>
/// <remarks>
/// <para>
/// Composes the same pipeline as the M2 <see cref="SyncCallEmissionEndToEndTests"/>:
/// in-memory <see cref="SqliteAuditWriter"/> + <see cref="RingBufferFallback"/> +
/// <see cref="FallbackAuditWriter"/> on the site, drained by a real
/// <see cref="SiteAuditTelemetryActor"/> through a
/// <see cref="DirectActorSiteStreamAuditClient"/> stub that short-circuits the
/// gRPC wire and Asks the central <see cref="AuditLogIngestActor"/> backed by
/// the real <see cref="AuditLogRepository"/> on the per-class
/// <see cref="MsSqlMigrationFixture"/> MSSQL database.
/// </para>
/// <para>
/// Drives the AuditingDbConnection wrapper directly via
/// <see cref="ScriptRuntimeContext.DatabaseHelper"/>'s internal ctor (the
/// AuditLog tests project has <c>InternalsVisibleTo</c> on SiteRuntime). No
/// script runtime, no Akka Instance Actor — the test wires the helper, opens
/// an in-memory SQLite connection through a stub <see cref="IDatabaseGateway"/>,
/// runs one SQL statement, and waits for the central row to land. Each test
/// uses a unique <c>SourceSiteId</c> (Guid suffix) so concurrent tests
/// sharing the MSSQL fixture don't interfere with each other.
/// </para>
/// </remarks>
public class DatabaseSyncEmissionEndToEndTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public DatabaseSyncEmissionEndToEndTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private const string ConnectionName = "machineData";
private const string InstanceName = "Plant.Pump42";
private const string SourceScript = "ScriptActor:doDbWork";
private static string NewSiteId() =>
"test-e1-db-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.Options;
return new ScadaBridgeDbContext(options);
}
/// <summary>
/// Per-test in-memory SQLite database with a tiny 2-row schema we can both
/// write to and select from. Mirrors the pattern from
/// <c>DatabaseSyncEmissionTests</c> — the keep-alive root keeps the
/// in-memory database file pinned for the duration of the test, while the
/// returned <c>live</c> connection is what the stub gateway hands back to
/// the auditing wrapper.
/// </summary>
private static SqliteConnection NewInMemoryDb(out SqliteConnection keepAlive)
{
var dbName = $"db-{Guid.NewGuid():N}";
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
keepAlive = new SqliteConnection(connStr);
keepAlive.Open();
using (var seed = keepAlive.CreateCommand())
{
seed.CommandText =
"CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);" +
"INSERT INTO t (id, name) VALUES (1, 'alpha');" +
"INSERT INTO t (id, name) VALUES (2, 'beta');";
seed.ExecuteNonQuery();
}
var live = new SqliteConnection(connStr);
live.Open();
return live;
}
private static SqliteAuditWriter CreateInMemorySqliteWriter() =>
new(
Options.Create(new SqliteAuditWriterOptions
{
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 1024,
}),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride:
$"Data Source=file:auditlog-e1-{Guid.NewGuid():N}?mode=memory&cache=shared");
private static IOptions<SiteAuditTelemetryOptions> FastTelemetryOptions() =>
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = 256,
// 1s on both intervals so the initial scheduled tick fires quickly
// — drains the SQLite Pending row and pushes it through the stub
// gRPC client into the central ingest actor.
BusyIntervalSeconds = 1,
IdleIntervalSeconds = 1,
});
private IActorRef CreateIngestActor(IAuditLogRepository repo) =>
Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
repo,
NullLogger<AuditLogIngestActor>.Instance)));
private IActorRef CreateTelemetryActor(
ISiteAuditQueue queue,
ISiteStreamAuditClient client) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
queue,
client,
FastTelemetryOptions(),
NullLogger<SiteAuditTelemetryActor>.Instance)));
/// <summary>
/// Wires the production
/// <see cref="ScriptRuntimeContext.DatabaseHelper"/> (internal ctor) onto
/// the supplied <see cref="IDatabaseGateway"/> + <see cref="IAuditWriter"/>
/// with the test's site id and source script. The returned helper's
/// <c>Connection(...)</c> hands back a real <c>AuditingDbConnection</c>.
/// </summary>
private static ScriptRuntimeContext.DatabaseHelper CreateHelper(
IDatabaseGateway gateway,
IAuditWriter writer,
string siteId) =>
new(
gateway,
InstanceName,
NullLogger.Instance,
Guid.NewGuid(),
auditWriter: writer,
siteId: siteId,
sourceScript: SourceScript,
cachedForwarder: null);
[SkippableFact]
public async Task DbWrite_Insert_Emits_OneCentralRow_WithExtraOpWrite_AndRowsAffected()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
// Central — repository + ingest actor backed by the MSSQL fixture.
await using var ingestContext = CreateContext();
var ingestRepo = new AuditLogRepository(ingestContext);
var ingestActor = CreateIngestActor(ingestRepo);
// Site — SQLite audit writer + ring + fallback + telemetry actor that
// drains into the stub gRPC client which forwards to the ingest actor.
await using var sqliteWriter = CreateInMemorySqliteWriter();
var ring = new RingBufferFallback();
var fallback = new FallbackAuditWriter(
sqliteWriter,
ring,
new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
CreateTelemetryActor(sqliteWriter, stubClient);
// SQLite-backed inner connection — the stub gateway hands it to the
// auditing wrapper as the DbConnection the script would have got.
using var keepAlive = new SqliteConnection("Data Source=k1;Mode=Memory;Cache=Shared");
var inner = NewInMemoryDb(out _);
var gateway = Substitute.For<IDatabaseGateway>();
gateway.GetConnectionAsync(ConnectionName, Arg.Any<CancellationToken>())
.Returns(inner);
// Act — one INSERT through the auditing wrapper. The wrapper emits a
// single DbOutbound/DbWrite event to the fallback writer; the
// telemetry actor's next tick drains it to central.
var helper = CreateHelper(gateway, fallback, siteId);
await using (var conn = await helper.Connection(ConnectionName))
await using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "INSERT INTO t (id, name) VALUES (3, 'gamma')";
var rows = await cmd.ExecuteNonQueryAsync();
Assert.Equal(1, rows);
}
// Assert — one central row, Kind=DbWrite, Status=Delivered,
// Extra.op="write", Extra.rowsAffected=1. 15s upper bound covers the
// initial 1s tick + SQLite drain + actor round-trip + EF/MSSQL latency.
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var readRepo = new AuditLogRepository(readContext);
var rows = await readRepo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 10));
var evt = Assert.Single(rows);
Assert.Equal(AuditChannel.DbOutbound, evt.Channel);
Assert.Equal(AuditKind.DbWrite, evt.Kind);
Assert.Equal(AuditStatus.Delivered, evt.Status);
Assert.Equal(siteId, evt.SourceSiteId);
Assert.Equal(InstanceName, evt.SourceInstanceId);
Assert.Equal(SourceScript, evt.SourceScript);
Assert.NotNull(evt.Extra);
Assert.Contains("\"op\":\"write\"", evt.Extra);
Assert.Contains("\"rowsAffected\":1", evt.Extra);
// Central stamps IngestedAtUtc; the site never sets it.
Assert.NotNull(evt.IngestedAtUtc);
Assert.StartsWith(ConnectionName, evt.Target);
}, TimeSpan.FromSeconds(15));
}
[SkippableFact]
public async Task DbWrite_Select_Emits_OneCentralRow_WithExtraOpRead_AndRowsReturned()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var ingestContext = CreateContext();
var ingestRepo = new AuditLogRepository(ingestContext);
var ingestActor = CreateIngestActor(ingestRepo);
await using var sqliteWriter = CreateInMemorySqliteWriter();
var ring = new RingBufferFallback();
var fallback = new FallbackAuditWriter(
sqliteWriter,
ring,
new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
CreateTelemetryActor(sqliteWriter, stubClient);
using var keepAlive = new SqliteConnection("Data Source=k2;Mode=Memory;Cache=Shared");
var inner = NewInMemoryDb(out _);
var gateway = Substitute.For<IDatabaseGateway>();
gateway.GetConnectionAsync(ConnectionName, Arg.Any<CancellationToken>())
.Returns(inner);
var helper = CreateHelper(gateway, fallback, siteId);
await using (var conn = await helper.Connection(ConnectionName))
await using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "SELECT id, name FROM t ORDER BY id";
await using var reader = await cmd.ExecuteReaderAsync();
var seen = 0;
while (await reader.ReadAsync())
{
seen++;
}
// Explicit close so the AuditingDbDataReader callback fires before
// the helper is disposed (Bundle A defers the audit emission to
// reader-close so rowsReturned is observable).
await reader.CloseAsync();
Assert.Equal(2, seen);
}
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var readRepo = new AuditLogRepository(readContext);
var rows = await readRepo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 10));
var evt = Assert.Single(rows);
Assert.Equal(AuditChannel.DbOutbound, evt.Channel);
Assert.Equal(AuditKind.DbWrite, evt.Kind);
Assert.Equal(AuditStatus.Delivered, evt.Status);
Assert.NotNull(evt.Extra);
Assert.Contains("\"op\":\"read\"", evt.Extra);
Assert.Contains("\"rowsReturned\":2", evt.Extra);
Assert.NotNull(evt.IngestedAtUtc);
}, TimeSpan.FromSeconds(15));
}
}
@@ -0,0 +1,276 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Audit Log #23 — ExecutionId end-to-end correlation suite verifying the
/// universal per-run correlation promise: <b>every audit row produced by one
/// script execution carries the same non-null <see cref="AuditEvent.ExecutionId"/></b>.
/// </summary>
/// <remarks>
/// <para>
/// This is the integration-level counterpart to the unit-level
/// <c>ExecutionCorrelationContextTests</c> in <c>ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests</c>:
/// where that test asserts the shared id on the in-memory captured rows, this
/// suite drives the rows all the way through the production pipeline — the real
/// <see cref="SqliteAuditWriter"/> site hot-path, the real
/// <see cref="SiteAuditTelemetryActor"/> drain loop, the real
/// <see cref="AuditLogIngestActor"/>, and the real <see cref="AuditLogRepository"/>
/// over the per-class <see cref="MsSqlMigrationFixture"/> MSSQL database — then
/// reads the rows back from the central store and asserts the shared id.
/// </para>
/// <para>
/// Composes the same pipeline as the M2 <see cref="SyncCallEmissionEndToEndTests"/>
/// and the M4 <see cref="DatabaseSyncEmissionEndToEndTests"/>: an in-memory
/// <see cref="SqliteAuditWriter"/> + <see cref="RingBufferFallback"/> +
/// <see cref="FallbackAuditWriter"/> on the site, drained by a real
/// <see cref="SiteAuditTelemetryActor"/> through the shared
/// <see cref="DirectActorSiteStreamAuditClient"/> stub that short-circuits the
/// gRPC wire and Asks the central ingest actor. The production
/// <see cref="ScriptRuntimeContext"/> is driven directly: one context performs
/// two distinct trust-boundary actions — a sync <c>ExternalSystem.Call</c> and a
/// sync <c>Database</c> write — so the two emitted audit rows originate from one
/// execution. Each test uses a unique <c>ExecutionId</c> + <c>SourceSiteId</c>
/// (Guid suffixes) so concurrent tests sharing the MSSQL fixture don't interfere.
/// </para>
/// </remarks>
public class ExecutionIdCorrelationTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public ExecutionIdCorrelationTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private const string ConnectionName = "machineData";
private const string InstanceName = "Plant.Pump42";
private const string SourceScript = "ScriptActor:OnTick";
private static string NewSiteId() =>
"test-execid-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.Options;
return new ScadaBridgeDbContext(options);
}
/// <summary>
/// Per-test in-memory SQLite database with a tiny single-table schema the
/// sync DB write targets. The keep-alive root pins the in-memory file for
/// the duration of the test; the returned <c>live</c> connection is what the
/// stub gateway hands back to the auditing wrapper. Mirrors
/// <c>DatabaseSyncEmissionEndToEndTests.NewInMemoryDb</c>.
/// </summary>
private static SqliteConnection NewInMemoryDb(out SqliteConnection keepAlive)
{
var dbName = $"db-{Guid.NewGuid():N}";
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
keepAlive = new SqliteConnection(connStr);
keepAlive.Open();
using (var seed = keepAlive.CreateCommand())
{
seed.CommandText =
"CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);";
seed.ExecuteNonQuery();
}
var live = new SqliteConnection(connStr);
live.Open();
return live;
}
private static SqliteAuditWriter CreateInMemorySqliteWriter() =>
new(
Options.Create(new SqliteAuditWriterOptions
{
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 1024,
}),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride:
$"Data Source=file:auditlog-execid-{Guid.NewGuid():N}?mode=memory&cache=shared");
private static IOptions<SiteAuditTelemetryOptions> FastTelemetryOptions() =>
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = 256,
// 1s on both intervals so the initial scheduled tick fires quickly
// — drains the SQLite Pending rows and pushes them through the stub
// gRPC client into the central ingest actor.
BusyIntervalSeconds = 1,
IdleIntervalSeconds = 1,
});
private IActorRef CreateIngestActor(IAuditLogRepository repo) =>
Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
repo,
NullLogger<AuditLogIngestActor>.Instance)));
private IActorRef CreateTelemetryActor(
ISiteAuditQueue queue,
ISiteStreamAuditClient client) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
queue,
client,
FastTelemetryOptions(),
NullLogger<SiteAuditTelemetryActor>.Instance)));
[SkippableFact]
public async Task OneExecution_ApiCallAndDbWrite_AllCentralRows_ShareOneNonNullExecutionId()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
// An explicit per-run execution id — the value the test asserts on every
// audit row produced by the single script execution below.
var executionId = Guid.NewGuid();
// ── Central — repository + ingest actor backed by the MSSQL fixture ──
await using var ingestContext = CreateContext();
var ingestRepo = new AuditLogRepository(ingestContext);
var ingestActor = CreateIngestActor(ingestRepo);
// ── Site — SQLite audit writer + ring + fallback + telemetry actor ───
await using var sqliteWriter = CreateInMemorySqliteWriter();
var ring = new RingBufferFallback();
var fallback = new FallbackAuditWriter(
sqliteWriter,
ring,
new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
CreateTelemetryActor(sqliteWriter, stubClient);
// Outbound API client — one successful CallAsync, one audit row.
var externalClient = Substitute.For<IExternalSystemClient>();
externalClient
.CallAsync("ERP", "GetOrder", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
.Returns(new ExternalCallResult(true, "{}", null));
// SQLite-backed inner DB connection — the stub gateway hands it to the
// auditing wrapper as the connection the script would have got.
using var keepAlive = new SqliteConnection("Data Source=execid-k1;Mode=Memory;Cache=Shared");
var innerDb = NewInMemoryDb(out _);
var gateway = Substitute.For<IDatabaseGateway>();
gateway.GetConnectionAsync(ConnectionName, Arg.Any<CancellationToken>())
.Returns(innerDb);
// ── Act — ONE script execution: a sync ExternalSystem.Call AND a sync
// Database write, both performed through a SINGLE ScriptRuntimeContext
// stamped with the explicit executionId. Each helper emits exactly one
// trust-boundary audit row to the fallback writer; the telemetry actor's
// next tick drains both to central.
var context = CreateScriptContext(externalClient, gateway, fallback, siteId, executionId);
await context.ExternalSystem.Call("ERP", "GetOrder");
await using (var conn = await context.Database.Connection(ConnectionName))
await using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "INSERT INTO t (id, name) VALUES (1, 'alpha')";
var affected = await cmd.ExecuteNonQueryAsync();
Assert.Equal(1, affected);
}
// ── Assert — read the rows back from the CENTRAL store filtered by the
// execution id; both the ApiCall and the DbWrite row must be present and
// every one must carry the SAME non-null ExecutionId we minted above.
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var readRepo = new AuditLogRepository(readContext);
// The ExecutionId filter dimension is the universal-correlation
// query an audit reader uses to pull every action of one run.
var rows = await readRepo.QueryAsync(
new AuditLogQueryFilter(ExecutionId: executionId),
new AuditLogPaging(PageSize: 10));
// Both trust-boundary actions of the one execution have landed.
Assert.Equal(2, rows.Count);
// Every central row carries the SAME non-null ExecutionId — the
// core promise of the per-run correlation value.
Assert.All(rows, r =>
{
Assert.NotNull(r.ExecutionId);
Assert.Equal(executionId, r.ExecutionId);
Assert.Equal(siteId, r.SourceSiteId);
// Central stamps IngestedAtUtc; the site never sets it.
Assert.NotNull(r.IngestedAtUtc);
});
// The two rows are the two distinct trust-boundary actions — one
// outbound API call and one outbound DB write — proving the shared
// id spans different channels, not two rows of the same action.
Assert.Single(rows, r => r.Channel == AuditChannel.ApiOutbound && r.Kind == AuditKind.ApiCall);
Assert.Single(rows, r => r.Channel == AuditChannel.DbOutbound && r.Kind == AuditKind.DbWrite);
}, TimeSpan.FromSeconds(15));
}
/// <summary>
/// Builds a production <see cref="ScriptRuntimeContext"/> wired with the
/// outbound external-system client, the database gateway and the audit
/// writer, stamped with an explicit <paramref name="executionId"/>. The
/// actor refs are <see cref="ActorRefs.Nobody"/> — the ExternalSystem /
/// Database helpers exercised here never touch them.
/// </summary>
private static ScriptRuntimeContext CreateScriptContext(
IExternalSystemClient externalSystemClient,
IDatabaseGateway databaseGateway,
IAuditWriter auditWriter,
string siteId,
Guid executionId)
{
var compilationService = new ScriptCompilationService(
NullLogger<ScriptCompilationService>.Instance);
var sharedScriptLibrary = new SharedScriptLibrary(
compilationService, NullLogger<SharedScriptLibrary>.Instance);
return new ScriptRuntimeContext(
ActorRefs.Nobody,
ActorRefs.Nobody,
sharedScriptLibrary,
currentCallDepth: 0,
maxCallDepth: 10,
askTimeout: TimeSpan.FromSeconds(5),
instanceName: InstanceName,
logger: NullLogger.Instance,
externalSystemClient: externalSystemClient,
databaseGateway: databaseGateway,
storeAndForward: null,
siteCommunicationActor: null,
siteId: siteId,
sourceScript: SourceScript,
auditWriter: auditWriter,
operationTrackingStore: null,
cachedForwarder: null,
executionId: executionId);
}
}
@@ -0,0 +1,298 @@
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
using System.Security.Claims;
using System.Text;
using System.Text.Encodings.Web;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Audit Log #23 — M4 Bundle E (Task E3) end-to-end audit trail for the
/// inbound API surface. Wires the production
/// <see cref="AuditWriteMiddleware"/> into a Microsoft.AspNetCore.TestHost
/// pipeline that mirrors the production
/// <c>UseAuthentication → UseAuditWriteMiddleware → POST /api/{methodName}</c>
/// order, with the real <see cref="CentralAuditWriter"/> backed by the per-class
/// <see cref="MsSqlMigrationFixture"/> MSSQL <c>AuditLog</c> table.
/// </summary>
/// <remarks>
/// <para>
/// Three response shapes are covered: a happy-path 200 (with the actor
/// resolved from <see cref="HttpContext.Items"/>), a 401 unauthenticated
/// (Actor stays null, kind flips to
/// <see cref="AuditKind.InboundAuthFailure"/>), and a 500 internal-error
/// response. Each test uses a unique method name so concurrent tests sharing
/// the fixture don't interfere.
/// </para>
/// <para>
/// The middleware-level unit tests already cover the recording-writer shape
/// (<c>AuditWriteMiddlewareTests</c>) and the pipeline ordering
/// (<c>MiddlewareOrderTests</c>); these tests verify the END-TO-END
/// materialisation in the central <c>AuditLog</c> table — the production
/// glue from request → writer → repository → MSSQL row.
/// </para>
/// </remarks>
public class InboundApiAuditTests : IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public InboundApiAuditTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
/// <summary>
/// Per-test unique method name suffix — the audit row's <c>Target</c>
/// captures it so each test can query by <see cref="AuditLogQueryFilter.Target"/>
/// without disturbing other tests using the same MSSQL fixture.
/// </summary>
private static string NewMethodName(string prefix) =>
prefix + "-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
.Options;
return new ScadaBridgeDbContext(options);
}
/// <summary>
/// Spins up a minimal in-memory ASP.NET host whose pipeline mirrors the
/// production arrangement. The endpoint handler delegate is supplied by
/// each test so it can shape the response (200 with an actor, 401
/// auth-fail, 500 server error) the way the production handler would.
/// </summary>
private async Task<IHost> BuildHostAsync(RequestDelegate endpointHandler)
{
var hostBuilder = new HostBuilder()
.ConfigureWebHost(webBuilder =>
{
webBuilder
.UseTestServer()
.ConfigureServices(services =>
{
// Real EF DbContext + AuditLogRepository wired against
// the per-class MSSQL fixture. CentralAuditWriter is a
// singleton — same pattern the production Host uses —
// opening a fresh scope per call to resolve the scoped
// repository.
services.AddDbContext<ScadaBridgeDbContext>(opts =>
opts.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
services.AddScoped<IAuditLogRepository>(sp =>
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
services.AddSingleton<ICentralAuditWriter>(sp =>
new CentralAuditWriter(sp, NullLogger<CentralAuditWriter>.Instance));
services.AddRouting();
services.AddAuthorization();
services.AddAuthentication("TestScheme")
.AddScheme<AuthenticationSchemeOptions, AlwaysAuthenticatedHandler>(
"TestScheme", _ => { });
})
.Configure(app =>
{
// Mirror production order: routing → auth → audit
// middleware → endpoint. The auth scheme always
// succeeds; per-request auth-failure semantics are
// produced INSIDE the endpoint handler (mirroring
// ApiKeyValidator's in-handler short-circuit).
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseAuditWriteMiddleware();
app.UseEndpoints(endpoints =>
{
endpoints.MapPost("/api/{methodName}", endpointHandler);
});
});
});
return await hostBuilder.StartAsync();
}
/// <summary>
/// Minimal authentication handler that always succeeds — keeps
/// <see cref="HttpContext.User"/> populated so the middleware's
/// Items-then-User fallback path has a real principal to ignore. The
/// middleware's primary actor resolution path uses
/// <see cref="AuditWriteMiddleware.AuditActorItemKey"/> so this handler's
/// claim never appears on the emitted Actor unless the endpoint stashes
/// it explicitly.
/// </summary>
private sealed class AlwaysAuthenticatedHandler : AuthenticationHandler<AuthenticationSchemeOptions>
{
public AlwaysAuthenticatedHandler(
Microsoft.Extensions.Options.IOptionsMonitor<AuthenticationSchemeOptions> options,
Microsoft.Extensions.Logging.ILoggerFactory logger,
UrlEncoder encoder)
: base(options, logger, encoder) { }
protected override Task<AuthenticateResult> HandleAuthenticateAsync()
{
var identity = new ClaimsIdentity(
new[] { new Claim(ClaimTypes.Name, "framework-user") }, "TestScheme");
var principal = new ClaimsPrincipal(identity);
var ticket = new AuthenticationTicket(principal, "TestScheme");
return Task.FromResult(AuthenticateResult.Success(ticket));
}
}
/// <summary>
/// Queries the central <c>AuditLog</c> table for the row produced by the
/// test's unique method name. Wrapped in <see cref="Assert"/> calls so the
/// query can be used inside a polling helper.
/// </summary>
private async Task<IReadOnlyList<AuditEvent>> QueryByTargetAsync(string methodName)
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
return await repo.QueryAsync(
new AuditLogQueryFilter(Target: methodName),
new AuditLogPaging(PageSize: 10));
}
/// <summary>
/// Awaits the central <c>AuditLog</c> row materialising for
/// <paramref name="methodName"/>. The writer is fire-and-forget so we
/// poll briefly after the HTTP response returns to absorb scheduling
/// jitter between the middleware's <c>finally</c> block and the row hitting
/// MSSQL.
/// </summary>
private async Task<AuditEvent> AwaitOneAsync(string methodName)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10);
while (DateTime.UtcNow < deadline)
{
var rows = await QueryByTargetAsync(methodName);
if (rows.Count > 0)
{
return Assert.Single(rows);
}
await Task.Delay(100);
}
// Fall through to a final query so the failure message carries the
// actual row count from the last attempt.
var finalRows = await QueryByTargetAsync(methodName);
return Assert.Single(finalRows);
}
[SkippableFact]
public async Task PostToApi_WithValidActor_Emits_InboundRequest_StatusDelivered_HttpStatus200_ActorPopulated()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var methodName = NewMethodName("echo");
using var host = await BuildHostAsync(async ctx =>
{
// Simulate the production endpoint stashing the resolved API key
// name on HttpContext.Items AFTER successful auth — the middleware
// reads it in its finally block to populate Actor.
ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "integration-svc";
ctx.Response.StatusCode = 200;
await ctx.Response.WriteAsync("ok");
});
var client = host.GetTestClient();
var resp = await client.PostAsync(
$"/api/{methodName}",
new StringContent("{\"x\":1}", Encoding.UTF8, "application/json"));
Assert.Equal(System.Net.HttpStatusCode.OK, resp.StatusCode);
var evt = await AwaitOneAsync(methodName);
Assert.Equal(AuditChannel.ApiInbound, evt.Channel);
Assert.Equal(AuditKind.InboundRequest, evt.Kind);
Assert.Equal(AuditStatus.Delivered, evt.Status);
Assert.Equal(200, evt.HttpStatus);
Assert.Equal("integration-svc", evt.Actor);
// Central direct-write — no site-local forward state (alog.md §6).
Assert.Null(evt.ForwardState);
// IngestedAtUtc stamped by the central writer.
Assert.NotNull(evt.IngestedAtUtc);
}
[SkippableFact]
public async Task PostToApi_Without_Auth_Emits_InboundAuthFailure_StatusFailed_HttpStatus401_ActorNull()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var methodName = NewMethodName("auth-fail");
using var host = await BuildHostAsync(async ctx =>
{
// The production ApiKeyValidator returns 401 from inside the
// handler when the X-API-Key header is missing or invalid; the
// handler must NOT stash an actor name in that case so the
// middleware emits Actor=null on the resulting audit row.
ctx.Response.StatusCode = 401;
await ctx.Response.WriteAsync("unauthorized");
});
var client = host.GetTestClient();
var resp = await client.PostAsync(
$"/api/{methodName}",
new StringContent("{}", Encoding.UTF8, "application/json"));
Assert.Equal(System.Net.HttpStatusCode.Unauthorized, resp.StatusCode);
var evt = await AwaitOneAsync(methodName);
Assert.Equal(AuditChannel.ApiInbound, evt.Channel);
Assert.Equal(AuditKind.InboundAuthFailure, evt.Kind);
Assert.Equal(AuditStatus.Failed, evt.Status);
Assert.Equal(401, evt.HttpStatus);
// Never echo back an unauthenticated principal — middleware suppresses
// the framework user resolution on 401/403 paths.
Assert.Null(evt.Actor);
}
[SkippableFact]
public async Task PostToApi_Returning500_Emits_InboundRequest_StatusFailed_HttpStatus500()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var methodName = NewMethodName("server-error");
using var host = await BuildHostAsync(async ctx =>
{
// A handler-returned 500 (not a throw) — auth succeeded so Actor
// resolution is still allowed; the audit row's Kind stays
// InboundRequest (not InboundAuthFailure) and Status flips to
// Failed because the response is not a 2xx.
ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "integration-svc";
ctx.Response.StatusCode = 500;
await ctx.Response.WriteAsync("kaboom");
});
var client = host.GetTestClient();
var resp = await client.PostAsync(
$"/api/{methodName}",
new StringContent("{}", Encoding.UTF8, "application/json"));
Assert.Equal(System.Net.HttpStatusCode.InternalServerError, resp.StatusCode);
var evt = await AwaitOneAsync(methodName);
Assert.Equal(AuditChannel.ApiInbound, evt.Channel);
Assert.Equal(AuditKind.InboundRequest, evt.Kind);
Assert.Equal(AuditStatus.Failed, evt.Status);
Assert.Equal(500, evt.HttpStatus);
Assert.Equal("integration-svc", evt.Actor);
}
}
@@ -0,0 +1,125 @@
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
using Google.Protobuf.WellKnownTypes;
using Timestamp = Google.Protobuf.WellKnownTypes.Timestamp;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
/// <summary>
/// Test-side combined-telemetry dispatcher: wraps a production
/// <see cref="ICachedCallTelemetryForwarder"/> so the local audit + tracking
/// stores still get written, then projects the same packet onto the wire as a
/// <see cref="CachedTelemetryBatch"/> and pushes it through the supplied
/// <see cref="ISiteStreamAuditClient"/>. The bridge can be composed into the
/// existing <see cref="CachedCallLifecycleBridge"/> chain as the
/// <see cref="ICachedCallTelemetryForwarder"/> implementation so a single
/// observer callback fans out to both halves.
/// </summary>
/// <remarks>
/// <para>
/// Production wiring keeps the wire push deferred to M6 — the site SQLite hot
/// path is the source of truth and a future M6 drain will push the rows
/// through the gRPC client. For end-to-end testing today we need a way to
/// exercise the central dual-write transaction immediately, so this
/// dispatcher synthesises the wire packet inline and round-trips it through
/// the stub client. The shape mirrors what the M6 drain will eventually emit.
/// </para>
/// <para>
/// <b>Best-effort:</b> both the inner forwarder call and the wire push are
/// wrapped in independent try/catch blocks. A thrown wire client doesn't
/// abort the local writes (the audit row is already in SQLite); a thrown
/// local forwarder doesn't abort the wire push (central still gets the
/// dual-write attempt).
/// </para>
/// </remarks>
public sealed class CombinedTelemetryDispatcher : ICachedCallTelemetryForwarder
{
private readonly ICachedCallTelemetryForwarder _inner;
private readonly ISiteStreamAuditClient _wireClient;
public CombinedTelemetryDispatcher(
ICachedCallTelemetryForwarder inner,
ISiteStreamAuditClient wireClient)
{
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
_wireClient = wireClient ?? throw new ArgumentNullException(nameof(wireClient));
}
/// <inheritdoc/>
public async Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(telemetry);
// Inner forwarder writes the audit row to SQLite + updates the
// tracking store. Best-effort — exceptions are already swallowed
// inside the production forwarder, but wrap defensively here too in
// case a test substitutes a stricter inner.
try
{
await _inner.ForwardAsync(telemetry, ct).ConfigureAwait(false);
}
catch
{
// Swallow — alog.md §7 best-effort contract.
}
// Project the same packet onto the wire and push it through the stub
// client. This is the bit a future M6 drain will subsume — until
// then the test wraps the two halves into one observer-driven step.
try
{
var batch = new CachedTelemetryBatch();
batch.Packets.Add(BuildPacket(telemetry));
await _wireClient.IngestCachedTelemetryAsync(batch, ct).ConfigureAwait(false);
}
catch
{
// Swallow — the audit row is still in SQLite for a future drain;
// the central row will materialise the next time the wire path
// is exercised (or via the M6 reconciliation pull).
}
}
private static CachedTelemetryPacket BuildPacket(CachedCallTelemetry telemetry)
{
return new CachedTelemetryPacket
{
AuditEvent = AuditEventDtoMapper.ToDto(telemetry.Audit),
Operational = ToOperationalDto(telemetry.Operational),
};
}
private static SiteCallOperationalDto ToOperationalDto(SiteCallOperational op)
{
var dto = new SiteCallOperationalDto
{
TrackedOperationId = op.TrackedOperationId.Value.ToString("D"),
Channel = op.Channel,
Target = op.Target,
SourceSite = op.SourceSite,
SourceNode = op.SourceNode ?? string.Empty,
Status = op.Status,
RetryCount = op.RetryCount,
LastError = op.LastError ?? string.Empty,
CreatedAtUtc = Timestamp.FromDateTime(EnsureUtc(op.CreatedAtUtc)),
UpdatedAtUtc = Timestamp.FromDateTime(EnsureUtc(op.UpdatedAtUtc)),
};
if (op.HttpStatus.HasValue)
{
dto.HttpStatus = op.HttpStatus.Value;
}
if (op.TerminalAtUtc.HasValue)
{
dto.TerminalAtUtc = Timestamp.FromDateTime(EnsureUtc(op.TerminalAtUtc.Value));
}
return dto;
}
private static DateTime EnsureUtc(DateTime value) =>
value.Kind == DateTimeKind.Utc
? value
: DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc);
}
@@ -0,0 +1,177 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
/// <summary>
/// Shared end-to-end harness for the M3 cached-call combined telemetry tests
/// (G2/G3/G4). Composes the full pipeline:
/// <list type="bullet">
/// <item><description>Site-local SQLite <see cref="SqliteAuditWriter"/> (in-memory) +
/// <see cref="RingBufferFallback"/> + <see cref="FallbackAuditWriter"/>.</description></item>
/// <item><description>Site-local SQLite <see cref="OperationTrackingStore"/> (in-memory).</description></item>
/// <item><description>Production <see cref="CachedCallTelemetryForwarder"/> wrapped by a
/// test-side <see cref="CombinedTelemetryDispatcher"/> that also pushes each
/// packet through the stub gRPC client.</description></item>
/// <item><description><see cref="CachedCallLifecycleBridge"/> wired to the
/// dispatcher so a single observer call fans out audit + tracking + wire.</description></item>
/// <item><description><see cref="DirectActorSiteStreamAuditClient"/> connected
/// to an <see cref="AuditLogIngestActor"/> backed by the real
/// <see cref="AuditLogRepository"/> + <see cref="SiteCallAuditRepository"/>
/// against the per-test <see cref="MsSqlMigrationFixture"/> database.</description></item>
/// </list>
/// </summary>
/// <remarks>
/// <para>
/// Disposal cleans up the in-memory SQLite stores. The Akka actor system is
/// owned by the calling <see cref="Akka.TestKit.Xunit2.TestKit"/>; the harness
/// only owns the ingest actor IActorRef and the underlying repositories'
/// DbContext lifecycle.
/// </para>
/// </remarks>
public sealed class CombinedTelemetryHarness : IAsyncDisposable
{
public SqliteAuditWriter SqliteWriter { get; }
public RingBufferFallback Ring { get; }
public FallbackAuditWriter FallbackWriter { get; }
public OperationTrackingStore TrackingStore { get; }
public CachedCallTelemetryForwarder InnerForwarder { get; }
public CombinedTelemetryDispatcher Dispatcher { get; }
public CachedCallLifecycleBridge Bridge { get; }
public DirectActorSiteStreamAuditClient StubClient { get; }
public IActorRef IngestActor { get; }
public IServiceProvider ServiceProvider { get; }
private readonly MsSqlMigrationFixture _fixture;
private bool _disposed;
public CombinedTelemetryHarness(
MsSqlMigrationFixture fixture,
Akka.TestKit.Xunit2.TestKit testKit,
Func<ScadaBridgeDbContext, ISiteCallAuditRepository>? siteCallRepoOverride = null)
{
_fixture = fixture ?? throw new ArgumentNullException(nameof(fixture));
ArgumentNullException.ThrowIfNull(testKit);
// Site SQLite — unique in-memory database per harness so tests don't share
// an audit queue. Mode=Memory + Cache=Shared keeps the file alive for the
// lifetime of the writer connection.
SqliteWriter = new SqliteAuditWriter(
Options.Create(new SqliteAuditWriterOptions
{
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 1024,
}),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride:
$"Data Source=file:cachedcall-g-{Guid.NewGuid():N}?mode=memory&cache=shared");
Ring = new RingBufferFallback();
FallbackWriter = new FallbackAuditWriter(
SqliteWriter, Ring, new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
TrackingStore = new OperationTrackingStore(
Options.Create(new OperationTrackingOptions
{
// Same shared-in-memory pattern as the audit writer.
ConnectionString =
$"Data Source=file:tracking-g-{Guid.NewGuid():N}?mode=memory&cache=shared",
}),
NullLogger<OperationTrackingStore>.Instance);
// Central wiring: real repositories backed by the MSSQL fixture's DB.
ServiceProvider = BuildCentralServiceProvider(siteCallRepoOverride);
IngestActor = testKit.Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
ServiceProvider,
NullLogger<AuditLogIngestActor>.Instance)));
StubClient = new DirectActorSiteStreamAuditClient(IngestActor);
// Production forwarder writes the local stores; the dispatcher wraps
// it to ALSO push the same packet to central via the stub client.
InnerForwarder = new CachedCallTelemetryForwarder(
FallbackWriter, TrackingStore, NullLogger<CachedCallTelemetryForwarder>.Instance);
Dispatcher = new CombinedTelemetryDispatcher(InnerForwarder, StubClient);
Bridge = new CachedCallLifecycleBridge(Dispatcher, NullLogger<CachedCallLifecycleBridge>.Instance);
}
/// <summary>
/// Convenience: emit the initial submit packet directly through the
/// dispatcher (the bridge's hooks fire only for S&amp;F retry-loop
/// attempts; submit-row emission happens at the script call site).
/// </summary>
public Task EmitSubmitAsync(CachedCallTelemetry submit, CancellationToken ct = default) =>
Dispatcher.ForwardAsync(submit, ct);
/// <summary>
/// Convenience: route a per-attempt or terminal outcome through the bridge.
/// </summary>
public Task EmitAttemptAsync(CachedCallAttemptContext context, CancellationToken ct = default) =>
Bridge.OnAttemptCompletedAsync(context, ct);
public ScadaBridgeDbContext CreateReadContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.Options;
return new ScadaBridgeDbContext(options);
}
private IServiceProvider BuildCentralServiceProvider(
Func<ScadaBridgeDbContext, ISiteCallAuditRepository>? siteCallRepoOverride)
{
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(opts =>
opts.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
services.AddScoped<IAuditLogRepository>(sp =>
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
if (siteCallRepoOverride is null)
{
services.AddScoped<ISiteCallAuditRepository>(sp =>
new SiteCallAuditRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
}
else
{
services.AddScoped(sp =>
siteCallRepoOverride(sp.GetRequiredService<ScadaBridgeDbContext>()));
}
return services.BuildServiceProvider();
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
_disposed = true;
await SqliteWriter.DisposeAsync().ConfigureAwait(false);
await TrackingStore.DisposeAsync().ConfigureAwait(false);
if (ServiceProvider is IAsyncDisposable asyncSp)
{
await asyncSp.DisposeAsync().ConfigureAwait(false);
}
else if (ServiceProvider is IDisposable sp)
{
sp.Dispose();
}
}
}
@@ -0,0 +1,150 @@
using Akka.Actor;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
/// <summary>
/// Shared component-level <see cref="ISiteStreamAuditClient"/> test double that
/// short-circuits the gRPC wire and forwards each batch directly to a central
/// <see cref="AuditLog.Central.AuditLogIngestActor"/> via Akka <see cref="Futures.Ask"/>.
/// Lives under <c>Integration/Infrastructure/</c> so both the M2 sync-call and
/// M3 cached-call end-to-end suites can reuse it.
/// </summary>
/// <remarks>
/// <para>
/// The class deliberately mirrors the production <c>SiteStreamGrpcServer</c>
/// flow: decode each DTO into the in-process entity, Ask the central ingest
/// actor with the matching Akka command, and convert the Akka reply's accepted
/// id list into the proto <see cref="IngestAck"/> the telemetry actor / forwarder
/// expects. The actor wiring (single-repository vs. <see cref="IServiceProvider"/>
/// ctor) lives in the central ingest actor itself — this stub just routes the
/// command.
/// </para>
/// <para>
/// <see cref="FailNextCallCount"/> arms a deterministic number of failures
/// before the stub recovers; it applies to BOTH RPCs because the M2 sync-call
/// retry behaviour and the M3 cached-telemetry retry behaviour share a single
/// SiteAuditTelemetryActor drain. Tests that need to differentiate per-RPC
/// failures should reach for a per-test wrapper rather than extending this
/// shared infrastructure.
/// </para>
/// </remarks>
public sealed class DirectActorSiteStreamAuditClient : ISiteStreamAuditClient
{
private readonly IActorRef _ingestActor;
private int _failsRemaining;
private int _callCount;
private int _cachedTelemetryCallCount;
public DirectActorSiteStreamAuditClient(IActorRef ingestActor)
{
_ingestActor = ingestActor ?? throw new ArgumentNullException(nameof(ingestActor));
}
/// <summary>
/// When &gt; 0, the next <c>FailNextCallCount</c> invocations of either
/// RPC throw to simulate a gRPC error; after that count is exhausted, calls
/// succeed normally.
/// </summary>
public int FailNextCallCount
{
get => _failsRemaining;
set => _failsRemaining = value;
}
/// <summary>
/// Total successful + failed invocations of <see cref="IngestAuditEventsAsync"/>.
/// </summary>
public int CallCount => Volatile.Read(ref _callCount);
/// <summary>
/// Total successful + failed invocations of <see cref="IngestCachedTelemetryAsync"/>.
/// Separate counter so cached-call tests can assert dispatch independently of
/// any sync-call traffic going through the same stub.
/// </summary>
public int CachedTelemetryCallCount => Volatile.Read(ref _cachedTelemetryCallCount);
public async Task<IngestAck> IngestAuditEventsAsync(AuditEventBatch batch, CancellationToken ct)
{
Interlocked.Increment(ref _callCount);
// Atomically consume one of the queued failures, if any. This lets the
// test arm a deterministic number of failures before the stub recovers.
if (Interlocked.Decrement(ref _failsRemaining) >= 0)
{
throw new InvalidOperationException("simulated gRPC failure for test");
}
// Clamp at -1 to keep the field bounded under many calls.
Interlocked.Exchange(ref _failsRemaining, -1);
// Decode the proto batch back into AuditEvent records — mirrors what
// SiteStreamGrpcServer does before dispatching to the ingest actor.
var events = new List<AuditEvent>(batch.Events.Count);
foreach (var dto in batch.Events)
{
events.Add(AuditEventDtoMapper.FromDto(dto));
}
// Ask the central actor; the reply carries the accepted EventIds.
var reply = await _ingestActor
.Ask<IngestAuditEventsReply>(
new IngestAuditEventsCommand(events),
TimeSpan.FromSeconds(10))
.ConfigureAwait(false);
var ack = new IngestAck();
foreach (var id in reply.AcceptedEventIds)
{
ack.AcceptedEventIds.Add(id.ToString());
}
return ack;
}
/// <summary>
/// M3 dual-write path: decode each <see cref="CachedTelemetryPacket"/> into
/// the paired (<see cref="AuditEvent"/>, <see cref="SiteCall"/>) entry and
/// Ask the central ingest actor with an <see cref="IngestCachedTelemetryCommand"/>.
/// The accepted EventIds returned by the actor's dual-write transaction map
/// back into the proto ack.
/// </summary>
/// <remarks>
/// Uses the shared <see cref="AuditEventDtoMapper.FromDto"/> for the audit half
/// and <see cref="SiteCallDtoMapper.FromDto"/> for the SiteCall half — the same
/// canonical mappers the production <c>SiteStreamGrpcServer</c> uses.
/// </remarks>
public async Task<IngestAck> IngestCachedTelemetryAsync(CachedTelemetryBatch batch, CancellationToken ct)
{
Interlocked.Increment(ref _cachedTelemetryCallCount);
if (Interlocked.Decrement(ref _failsRemaining) >= 0)
{
throw new InvalidOperationException("simulated gRPC failure for test");
}
Interlocked.Exchange(ref _failsRemaining, -1);
var entries = new List<CachedTelemetryEntry>(batch.Packets.Count);
foreach (var packet in batch.Packets)
{
var audit = AuditEventDtoMapper.FromDto(packet.AuditEvent);
var siteCall = SiteCallDtoMapper.FromDto(packet.Operational);
entries.Add(new CachedTelemetryEntry(audit, siteCall));
}
var reply = await _ingestActor
.Ask<IngestCachedTelemetryReply>(
new IngestCachedTelemetryCommand(entries),
TimeSpan.FromSeconds(10))
.ConfigureAwait(false);
var ack = new IngestAck();
foreach (var id in reply.AcceptedEventIds)
{
ack.AcceptedEventIds.Add(id.ToString());
}
return ack;
}
}
@@ -0,0 +1,352 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Audit Log #23 — M4 Bundle E (Task E2): end-to-end audit trail produced by
/// the central <see cref="NotificationOutboxActor"/> dispatcher loop. Wires
/// the production <see cref="CentralAuditWriter"/> onto the real
/// <see cref="AuditLogRepository"/> against the per-class
/// <see cref="MsSqlMigrationFixture"/> MSSQL database, drives the dispatcher
/// with a stub <see cref="INotificationDeliveryAdapter"/> that yields a
/// transient-then-success sequence, and asserts the resulting
/// <see cref="AuditChannel.Notification"/>/<see cref="AuditKind.NotifyDeliver"/>
/// rows materialise with the expected Attempted/Delivered shape.
/// </summary>
/// <remarks>
/// <para>
/// The Submit row is normally produced by the site-side <c>Notify.Send</c>
/// wrapper (Bundle C); for this E2E we pre-insert a single AuditLog Submit row
/// via <see cref="IAuditLogRepository"/> alongside the seeded
/// <see cref="Notification"/> row so the assertions can confirm the dispatcher
/// emissions slot in alongside it. This keeps the test focused on the
/// dispatcher's emission shape without depending on the upstream site path.
/// </para>
/// <para>
/// Each test uses a unique notification id + source-site id so concurrent
/// tests sharing the MSSQL fixture don't interfere. The dispatcher is driven
/// deterministically via the internal
/// <c>InternalMessages.DispatchTick.Instance</c> sentinel (same pattern the
/// existing NotificationOutbox.Tests use).
/// </para>
/// </remarks>
public class NotifyDispatcherAuditTrailTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public NotifyDispatcherAuditTrailTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private static string NewSiteId() =>
"test-e2-notify-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
.Options;
return new ScadaBridgeDbContext(options);
}
/// <summary>
/// Builds a DI provider that mirrors the production wiring expected by
/// <see cref="NotificationOutboxActor"/>: scoped EF-backed
/// <see cref="INotificationOutboxRepository"/> + <see cref="INotificationRepository"/>
/// + the supplied <see cref="INotificationDeliveryAdapter"/>. The
/// <see cref="IAuditLogRepository"/> registration powers the
/// <see cref="CentralAuditWriter"/> the actor will emit through.
/// </summary>
private IServiceProvider BuildServiceProvider(INotificationDeliveryAdapter adapter)
{
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(opts =>
opts.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
services.AddScoped<INotificationOutboxRepository>(sp =>
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
services.AddScoped<INotificationRepository>(sp =>
new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
services.AddScoped<IAuditLogRepository>(sp =>
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
return services.BuildServiceProvider();
}
/// <summary>
/// Stub adapter that yields the next outcome from a configurable queue per
/// call. Lets a single dispatch sweep exercise the transient-then-success
/// transition by alternating <see cref="DeliveryResult.TransientFailure"/>
/// and <see cref="DeliveryResult.Success"/>.
/// </summary>
private sealed class QueuedOutcomeAdapter : INotificationDeliveryAdapter
{
private readonly Queue<DeliveryOutcome> _outcomes;
public int CallCount;
public QueuedOutcomeAdapter(params DeliveryOutcome[] outcomes)
{
_outcomes = new Queue<DeliveryOutcome>(outcomes);
}
public NotificationType Type => NotificationType.Email;
public Task<DeliveryOutcome> DeliverAsync(
Notification notification, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref CallCount);
// Defensive — if a test under-supplies outcomes we surface the
// problem as an explicit transient failure rather than throwing
// (the dispatcher would log + skip the notification but the audit
// assertions would be misleading).
var outcome = _outcomes.Count > 0
? _outcomes.Dequeue()
: DeliveryOutcome.Transient("test stub out of outcomes");
return Task.FromResult(outcome);
}
}
/// <summary>
/// Inserts a single SMTP configuration row so the dispatcher's
/// <c>ResolveRetryPolicyAsync</c> sees a real (maxRetries, retryDelay)
/// pair rather than the conservative fallback. A tiny positive RetryDelay
/// means a transient outcome's <c>NextAttemptAt</c> is immediately due —
/// useful so the SECOND DispatchTick re-claims the row without waiting.
/// NO-002: the dispatcher now clamps a non-positive RetryDelay to the
/// 1-minute fallback to avoid burn-looping on transient failures, so this
/// must be a strictly positive value (1 ms is fine for tests).
/// </summary>
private async Task SeedSmtpConfigAsync(int maxRetries = 5)
{
await using var ctx = CreateContext();
ctx.SmtpConfigurations.Add(new SmtpConfiguration(
"smtp.example.com", "Basic", "noreply@example.com")
{
MaxRetries = maxRetries,
RetryDelay = TimeSpan.FromMilliseconds(1),
});
await ctx.SaveChangesAsync();
}
/// <summary>
/// Seeds the Pending outbox row the dispatcher will claim. Using a fixed
/// caller-supplied <c>notificationId</c> so the test can later query the
/// AuditLog by <see cref="AuditEvent.CorrelationId"/> = notificationId.
/// </summary>
private async Task<Notification> SeedNotificationAsync(
Guid notificationId, string siteId, string listName = "ops-team")
{
await using var ctx = CreateContext();
var n = new Notification(
notificationId.ToString("D"),
NotificationType.Email,
listName,
"Tank overflow",
"Tank 3 level critical",
siteId)
{
SourceInstanceId = "Plant.Pump42",
SourceScript = "AlarmScript",
CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1),
};
ctx.Notifications.Add(n);
await ctx.SaveChangesAsync();
return n;
}
/// <summary>
/// Pre-inserts the Submit AuditLog row that the site-side Notify.Send
/// wrapper would have emitted (Bundle C). Keeps the assertions on the
/// dispatcher emissions intact without depending on the upstream site
/// path.
/// </summary>
private async Task SeedSubmitAuditRowAsync(Guid notificationId, string siteId)
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
var submitEvt = new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow.AddMinutes(-1),
Channel = AuditChannel.Notification,
Kind = AuditKind.NotifySend,
CorrelationId = notificationId,
SourceSiteId = siteId,
SourceInstanceId = "Plant.Pump42",
SourceScript = "AlarmScript",
Target = "ops-team",
Status = AuditStatus.Submitted,
ForwardState = AuditForwardState.Forwarded,
IngestedAtUtc = DateTime.UtcNow.AddMinutes(-1),
};
await repo.InsertIfNotExistsAsync(submitEvt);
}
private static NotificationOutboxOptions LongDispatchOptions() =>
// 1h dispatch + 24h purge so PreStart's timers never fire during the
// test; the test drives the dispatcher with explicit DispatchTick.
new()
{
DispatchInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromDays(1),
};
[SkippableFact]
public async Task NotifyDispatcher_FailThenSuccess_Emits_TwoAttempts_OneDelivered_Terminal()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var notificationId = Guid.NewGuid();
await SeedSmtpConfigAsync(maxRetries: 5);
await SeedNotificationAsync(notificationId, siteId);
await SeedSubmitAuditRowAsync(notificationId, siteId);
var adapter = new QueuedOutcomeAdapter(
DeliveryOutcome.Transient("smtp 421 try again"),
DeliveryOutcome.Success("ops@example.com"));
var serviceProvider = BuildServiceProvider(adapter);
var auditWriter = new CentralAuditWriter(
serviceProvider,
NullLogger<CentralAuditWriter>.Instance);
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
serviceProvider,
LongDispatchOptions(),
(ICentralAuditWriter)auditWriter,
NullLogger<NotificationOutboxActor>.Instance)));
// First tick: transient failure → one Attempted row, no terminal row.
actor.Tell(InternalMessages.DispatchTick.Instance);
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
var rows = await repo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 50));
// 1 Submit + 1 Attempted = 2 rows so far.
Assert.Equal(2, rows.Count);
Assert.Single(rows, r => r.Kind == AuditKind.NotifyDeliver
&& r.Status == AuditStatus.Attempted);
Assert.Single(rows, r => r.Kind == AuditKind.NotifySend);
}, TimeSpan.FromSeconds(15));
// Second tick: success → second Attempted + one Delivered terminal.
actor.Tell(InternalMessages.DispatchTick.Instance);
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
var rows = await repo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 50));
// 1 Submit + 2 Attempted + 1 Delivered terminal = 4 rows.
Assert.InRange(rows.Count, 3, 4);
var notifyDeliverRows = rows
.Where(r => r.Kind == AuditKind.NotifyDeliver)
.ToList();
Assert.Equal(2, notifyDeliverRows.Count(r => r.Status == AuditStatus.Attempted));
var terminal = Assert.Single(notifyDeliverRows, r => r.Status == AuditStatus.Delivered);
// All NotifyDeliver rows correlate to the original notification id.
Assert.All(notifyDeliverRows, r => Assert.Equal(notificationId, r.CorrelationId));
Assert.Equal("ops-team", terminal.Target);
}, TimeSpan.FromSeconds(15));
// Operational Notifications table mirrors the audit outcome.
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var n = await ctx.Notifications.SingleAsync(
row => row.NotificationId == notificationId.ToString("D"));
Assert.Equal(NotificationStatus.Delivered, n.Status);
Assert.NotNull(n.DeliveredAt);
}, TimeSpan.FromSeconds(15));
}
[SkippableFact]
public async Task NotifyDispatcher_AuditWriter_Throws_DeliveryStillSucceeds()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
var notificationId = Guid.NewGuid();
await SeedSmtpConfigAsync(maxRetries: 5);
await SeedNotificationAsync(notificationId, siteId);
var adapter = new QueuedOutcomeAdapter(
DeliveryOutcome.Success("ops@example.com"));
var serviceProvider = BuildServiceProvider(adapter);
// ALWAYS-throw writer wired in place of the production
// CentralAuditWriter. The dispatcher MUST still deliver the
// notification and persist the terminal Delivered transition
// regardless of the audit subsystem being down (alog.md §13).
var throwingWriter = new ThrowingCentralAuditWriter();
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
serviceProvider,
LongDispatchOptions(),
(ICentralAuditWriter)throwingWriter,
NullLogger<NotificationOutboxActor>.Instance)));
actor.Tell(InternalMessages.DispatchTick.Instance);
// The Notifications table is the operational source of truth — assert
// it transitions to Delivered even though every audit write threw.
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var n = await ctx.Notifications.SingleAsync(
row => row.NotificationId == notificationId.ToString("D"));
Assert.Equal(NotificationStatus.Delivered, n.Status);
Assert.NotNull(n.DeliveredAt);
}, TimeSpan.FromSeconds(15));
// The writer was attempted (at least once for the Attempted row, plus
// once for the Delivered terminal) — proves the dispatcher tried to
// emit and absorbed the throws rather than aborting the action.
Assert.True(throwingWriter.AttemptCount >= 2,
$"Expected the dispatcher to attempt audit writes; saw {throwingWriter.AttemptCount}");
}
/// <summary>
/// Test-only <see cref="ICentralAuditWriter"/> that ALWAYS throws on
/// <see cref="WriteAsync"/>. Used to verify the dispatcher's defensive
/// try/catch contract (alog.md §13) — audit failures must NEVER abort
/// the user-facing notification delivery.
/// </summary>
private sealed class ThrowingCentralAuditWriter : ICentralAuditWriter
{
private int _attemptCount;
public int AttemptCount => Volatile.Read(ref _attemptCount);
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
{
Interlocked.Increment(ref _attemptCount);
throw new InvalidOperationException(
"test-only ThrowingCentralAuditWriter — audit subsystem unavailable");
}
}
}
@@ -0,0 +1,351 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle F (#23 M6-T10) end-to-end test for the central-outage + reconciliation
/// recovery loop. Wires the real site SQLite hot-path
/// (<see cref="SqliteAuditWriter"/>) and the central <see cref="SiteAuditReconciliationActor"/>
/// with an <see cref="AuditLogIngestActor"/> backed by the real
/// <see cref="AuditLogRepository"/> on the per-test <see cref="MsSqlMigrationFixture"/>.
/// </summary>
/// <remarks>
/// <para>
/// The push path is deliberately omitted here: the brief models a sustained
/// central outage where the site queue grows unbounded in Pending, then a
/// reconciliation pull eventually drains everything once central comes back.
/// We reuse the production <see cref="IPullAuditEventsClient"/> seam (Bundle B)
/// with a test-only stub that wraps the same <see cref="ISiteAuditQueue.ReadPendingSinceAsync"/>
/// surface a real central-side gRPC client would hit, so the test is exercising
/// the actor's pull/ingest/mark-reconciled state machine end-to-end against
/// the real repository.
/// </para>
/// <para>
/// The <see cref="CombinedTelemetryHarness"/> from M3 is push-only — it has no
/// reconciliation puller — so we build the smaller stub inline rather than
/// retrofitting the shared harness with a code path it doesn't otherwise
/// need.
/// </para>
/// </remarks>
public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public OutageReconciliationTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
/// <summary>
/// Test-only <see cref="IPullAuditEventsClient"/> that mirrors how the
/// production central-side gRPC client will hit the site: read a batch
/// from <see cref="ISiteAuditQueue.ReadPendingSinceAsync"/>, then commit
/// via <see cref="ISiteAuditQueue.MarkReconciledAsync"/> once the central
/// repository accepts the rows. The Ask-based central path is wired by
/// the caller — we just expose the queue surface.
/// </summary>
/// <remarks>
/// The production wire shape will be:
/// central PullAuditEvents RPC → site SiteStreamGrpcServer.PullAuditEvents
/// → ISiteAuditQueue.ReadPendingSinceAsync → marshal proto → reply
/// followed by central InsertIfNotExistsAsync per row, then the site flips
/// the row to Reconciled on the next pull cycle. The stub collapses the
/// two halves (pull + commit) because the actor under test (the
/// reconciliation actor) is the side that drives both via the
/// IPullAuditEventsClient seam — committing back to the site after the
/// repository write is the reconciliation-actor invariant we want to
/// observe end-to-end.
/// </remarks>
private sealed class QueueBackedPullClient : IPullAuditEventsClient
{
private readonly ISiteAuditQueue _siteQueue;
public int CallCount { get; private set; }
public QueueBackedPullClient(ISiteAuditQueue siteQueue)
{
_siteQueue = siteQueue ?? throw new ArgumentNullException(nameof(siteQueue));
}
public async Task<PullAuditEventsResponse> PullAsync(
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
{
CallCount++;
var rows = await _siteQueue
.ReadPendingSinceAsync(sinceUtc, batchSize, ct)
.ConfigureAwait(false);
// Commit immediately on the site side — once the actor has the
// batch in hand it will InsertIfNotExistsAsync centrally; if the
// central insert later throws on a specific row, idempotency
// guarantees the next pull cycle does NOT re-fetch the row (it's
// already Reconciled on the site) but also does not surface the
// failure here. The brief calls this "ack-after-persist" — the
// production gRPC server will flip to Reconciled inside its
// PullAuditEvents handler after the central side has acknowledged
// (per Bundle A's race-fix, central is idempotent on EventId).
//
// MoreAvailable is true iff the read filled the batch — the actor
// uses this to decide whether to follow up on the next tick.
if (rows.Count > 0)
{
var ids = rows.Select(e => e.EventId).ToList();
await _siteQueue.MarkReconciledAsync(ids, ct).ConfigureAwait(false);
}
return new PullAuditEventsResponse(rows, MoreAvailable: rows.Count >= batchSize);
}
}
/// <summary>
/// In-memory enumerator returning a fixed single-site list — mirrors the
/// pattern used in <c>SiteAuditReconciliationActorTests</c>.
/// </summary>
private sealed class StaticEnumerator : ISiteEnumerator
{
private readonly IReadOnlyList<SiteEntry> _sites;
public StaticEnumerator(params SiteEntry[] sites) => _sites = sites;
public Task<IReadOnlyList<SiteEntry>> EnumerateAsync(CancellationToken ct = default) =>
Task.FromResult(_sites);
}
private ScadaBridgeDbContext CreateContext() =>
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString).Options);
private static AuditEvent NewEvent(string siteId, DateTime occurredAt) => new()
{
EventId = Guid.NewGuid(),
OccurredAtUtc = occurredAt,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = siteId,
Target = "external-system-a/method",
};
private SqliteAuditWriter CreateInMemorySqliteWriter() =>
new SqliteAuditWriter(
Options.Create(new SqliteAuditWriterOptions
{
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 4096,
}),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride:
$"Data Source=file:outage-{Guid.NewGuid():N}?mode=memory&cache=shared");
private (IServiceProvider Sp, IActorRef Ingest) BuildCentralPipeline()
{
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(opts =>
opts.UseSqlServer(_fixture.ConnectionString));
services.AddScoped<IAuditLogRepository>(sp =>
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
var sp = services.BuildServiceProvider();
var ingest = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
sp,
NullLogger<AuditLogIngestActor>.Instance)));
return (sp, ingest);
}
private static SiteAuditReconciliationOptions FastTickOptions(int batchSize = 256) => new()
{
ReconciliationIntervalSeconds = 300,
ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(100),
BatchSize = batchSize,
StalledAfterNonDrainingCycles = 2,
};
// ---------------------------------------------------------------------
// 1. CentralOutage_200Events_Buffer_Then_Reconciliation_Catches_Up_NoDuplicates
// ---------------------------------------------------------------------
[SkippableFact]
public async Task CentralOutage_200Events_Buffer_Then_Reconciliation_Catches_Up_NoDuplicates()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = "outage-recon-" + Guid.NewGuid().ToString("N").Substring(0, 8);
// Step 1: site accumulates 200 audit events during the simulated
// central outage. The push path is NOT wired here — every row stays
// Pending in the site SQLite store until reconciliation runs.
await using var sqliteWriter = CreateInMemorySqliteWriter();
var baseOccurred = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
const int totalEvents = 200;
var written = new List<AuditEvent>(totalEvents);
for (int i = 0; i < totalEvents; i++)
{
// Strictly monotonic OccurredAtUtc so the cursor can advance
// deterministically batch-by-batch — mirrors how a real script
// workload generates timestamps in wall-clock order.
var evt = NewEvent(siteId, baseOccurred.AddMilliseconds(i));
written.Add(evt);
await sqliteWriter.WriteAsync(evt);
}
// Sanity: every row is Pending (no push path wired, so nothing has
// been Forwarded or Reconciled yet).
var pending = await sqliteWriter.ReadPendingAsync(totalEvents + 10);
Assert.Equal(totalEvents, pending.Count);
// Step 2: central comes online — wire the ingest actor + reconciliation
// actor. The pull client wraps the site queue directly (the production
// shape is one RPC call); each pull advances the actor's cursor and
// flips rows on the site to Reconciled.
var (sp, ingest) = BuildCentralPipeline();
await using (sp as IAsyncDisposable ?? throw new InvalidOperationException())
{
var pullClient = new QueueBackedPullClient(sqliteWriter);
var enumerator = new StaticEnumerator(new SiteEntry(siteId, "http://test:8083"));
// BatchSize = 64 so the actor needs ~4 ticks to drain 200 rows.
// The "after 5 minutes" wording in the brief is satisfied by the
// fast-tick override (100 ms per tick) plus AwaitAssert giving
// the actor up to ~30 seconds to settle in real time.
var opts = FastTickOptions(batchSize: 64);
// Standalone DI scope for the reconciliation actor (it shares the
// ingest actor's IServiceProvider so both writers see the same
// EF context configuration).
var reconciliationActor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
enumerator,
pullClient,
sp,
Options.Create(opts),
NullLogger<SiteAuditReconciliationActor>.Instance)));
// Step 3: assert central AuditLog has all 200 rows after the
// actor drains. Polling the real MSSQL repository — the test
// fixture has its own database so a count restricted to this
// SourceSiteId is exact.
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var count = await ctx.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.CountAsync();
Assert.Equal(totalEvents, count);
},
duration: TimeSpan.FromSeconds(30),
interval: TimeSpan.FromMilliseconds(200));
// Step 4: assert site rows flipped to Reconciled.
// ReadPendingAsync only returns Pending rows; after a full drain
// it must be empty.
await AwaitAssertAsync(async () =>
{
var stillPending = await sqliteWriter.ReadPendingAsync(totalEvents + 10);
Assert.Empty(stillPending);
},
duration: TimeSpan.FromSeconds(10),
interval: TimeSpan.FromMilliseconds(100));
// Step 5: assert no duplicates by EventId — central must have
// exactly the 200 rows we wrote at the site (one row per EventId).
await using var verify = CreateContext();
var centralIds = await verify.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.Select(e => e.EventId)
.ToListAsync();
Assert.Equal(totalEvents, centralIds.Count);
Assert.Equal(totalEvents, centralIds.Distinct().Count());
// And every EventId we wrote at the site is present centrally.
Assert.True(written.All(w => centralIds.Contains(w.EventId)),
"every site-written EventId should be present centrally.");
// Tear the actor down before disposing the harness; the actor's
// PostStop cancels its scheduled timer.
Sys.Stop(reconciliationActor);
}
}
// ---------------------------------------------------------------------
// 2. ReconciliationPull_Idempotent_Across_Two_Cycles
// ---------------------------------------------------------------------
[SkippableFact]
public async Task ReconciliationPull_Idempotent_Across_Two_Cycles()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = "outage-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
const int totalEvents = 50;
await using var sqliteWriter = CreateInMemorySqliteWriter();
var baseOccurred = new DateTime(2026, 5, 20, 13, 0, 0, DateTimeKind.Utc);
for (int i = 0; i < totalEvents; i++)
{
await sqliteWriter.WriteAsync(NewEvent(siteId, baseOccurred.AddMilliseconds(i)));
}
var (sp, _) = BuildCentralPipeline();
await using (sp as IAsyncDisposable ?? throw new InvalidOperationException())
{
var pullClient = new QueueBackedPullClient(sqliteWriter);
var enumerator = new StaticEnumerator(new SiteEntry(siteId, "http://test:8083"));
var reconciliationActor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
enumerator,
pullClient,
sp,
Options.Create(FastTickOptions()),
NullLogger<SiteAuditReconciliationActor>.Instance)));
// Wait for the first drain cycle to complete.
await AwaitAssertAsync(async () =>
{
await using var ctx = CreateContext();
var count = await ctx.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.CountAsync();
Assert.Equal(totalEvents, count);
},
duration: TimeSpan.FromSeconds(30),
interval: TimeSpan.FromMilliseconds(200));
// Wait for additional pull cycles to fire — the actor ticks every
// 100 ms so a 1 s settle leaves the actor with at least ~5 ticks
// past the initial drain. Each subsequent tick must be a no-op
// because every row is now Reconciled and outside the
// ReadPendingSinceAsync filter.
var callsAfterDrain = pullClient.CallCount;
await Task.Delay(TimeSpan.FromMilliseconds(800));
Assert.True(pullClient.CallCount > callsAfterDrain,
$"expected additional pull calls after drain to validate idempotency, got {pullClient.CallCount} after {callsAfterDrain}");
// Central count must still be exactly totalEvents — no duplicates
// even though the cursor + read-Reconciled-too semantics could
// theoretically re-fetch on the second cycle.
await using var verify = CreateContext();
var rows = await verify.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(totalEvents, rows.Count);
Assert.Equal(totalEvents, rows.Select(r => r.EventId).Distinct().Count());
Sys.Stop(reconciliationActor);
}
}
}
@@ -0,0 +1,626 @@
using System.Text;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.InboundAPI;
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Audit Log #23 — <b>ParentExecutionId cross-execution correlation</b> headline
/// end-to-end suite. Verifies the inbound-API → routed-site-script bridge: an
/// inbound HTTP request runs an inbound method script that calls
/// <c>Route.Call</c> into a site instance; the routed site script does a sync
/// <c>ExternalSystem.Call</c>, a cached call and a <c>Notify.Send</c>. Every
/// audit row the routed run produces — site + central, sync + cached lifecycle
/// + <c>NotifySend</c>/<c>NotifyDeliver</c> — must carry
/// <see cref="AuditEvent.ParentExecutionId"/> equal to the inbound request's
/// <see cref="AuditEvent.ExecutionId"/>, while the routed run has its own
/// distinct <see cref="AuditEvent.ExecutionId"/> and the inbound
/// <see cref="AuditKind.InboundRequest"/> row is top-level
/// (<c>ParentExecutionId = NULL</c>).
/// </summary>
/// <remarks>
/// <para>
/// This is the integration-level counterpart to <see cref="ExecutionIdCorrelationTests"/>:
/// where that suite drives a single <see cref="ScriptRuntimeContext"/> run and
/// asserts the shared per-run <c>ExecutionId</c>, this suite spans <b>two</b>
/// executions on opposite sides of the inbound→routed bridge and asserts the
/// cross-execution <c>ParentExecutionId</c> linkage plus
/// <see cref="IAuditLogRepository.GetExecutionTreeAsync"/>.
/// </para>
/// <para>
/// The bridge is exercised through the genuine production glue:
/// <list type="bullet">
/// <item><description>the real <see cref="AuditWriteMiddleware"/> in a
/// Microsoft.AspNetCore.TestHost pipeline — mints the inbound request's
/// per-request <c>ExecutionId</c> once, stashes it on
/// <see cref="HttpContext.Items"/>, and emits the top-level
/// <see cref="AuditKind.InboundRequest"/> row via the real
/// <see cref="CentralAuditWriter"/>;</description></item>
/// <item><description>the real <see cref="InboundScriptExecutor"/> +
/// <see cref="RouteHelper"/> — the executor binds the stashed inbound
/// <c>ExecutionId</c> via <see cref="RouteHelper.WithParentExecutionId"/>, so a
/// <c>Route.To(...).Call(...)</c> inside the inbound script builds a
/// <see cref="RouteToCallRequest"/> carrying
/// <see cref="RouteToCallRequest.ParentExecutionId"/>.</description></item>
/// </list>
/// Only the cross-cluster routing transport is substituted: the test
/// <see cref="BridgingInstanceRouter"/> stands in for
/// <c>CommunicationServiceInstanceRouter</c> exactly as the production site
/// (<c>DeploymentManagerActor</c> → <c>ScriptActor</c> → <c>ScriptExecutionActor</c>)
/// would — it reads <see cref="RouteToCallRequest.ParentExecutionId"/> off the
/// wire request and threads it into the routed <see cref="ScriptRuntimeContext"/>
/// as <c>parentExecutionId</c>. A multi-node cluster is out of scope for an
/// in-process test (mirroring <c>SiteAuditPushFlowTests</c>'s relay).
/// </para>
/// <para>
/// The central audit store is the real <see cref="AuditLogRepository"/> over the
/// per-class <see cref="MsSqlMigrationFixture"/> MSSQL database; the routed run's
/// site rows reach it through the real <see cref="SqliteAuditWriter"/> hot-path +
/// <see cref="SiteAuditTelemetryActor"/> drain, the cached lifecycle rows through
/// the production <see cref="CachedCallTelemetryForwarder"/>, and the
/// <c>NotifyDeliver</c> rows through the real central
/// <see cref="NotificationOutboxActor"/> dispatcher.
/// </para>
/// </remarks>
public class ParentExecutionIdCorrelationTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public ParentExecutionIdCorrelationTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private const string RoutedInstanceCode = "Plant.Pump42";
private const string RoutedScriptName = "OnInboundRouted";
private const string ExternalSystemName = "ERP";
private const string ExternalMethodName = "GetOrder";
private const string NotifyListName = "ops-team";
/// <summary>Per-run site id (Guid suffix) so concurrent tests sharing the MSSQL fixture stay isolated.</summary>
private static string NewSiteId() =>
"test-parentexec-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
.Options;
return new ScadaBridgeDbContext(options);
}
[SkippableFact]
public async Task InboundRoutedRun_AllRoutedRows_CarryInboundExecutionId_AsParentExecutionId()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
// ── Central — repository + ingest actor + audit writer over the MSSQL fixture ──
var centralServices = new ServiceCollection();
centralServices.AddDbContext<ScadaBridgeDbContext>(opts =>
opts.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
centralServices.AddScoped<IAuditLogRepository>(sp =>
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
centralServices.AddScoped<ISiteCallAuditRepository>(sp =>
new SiteCallAuditRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
centralServices.AddScoped<INotificationOutboxRepository>(sp =>
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
centralServices.AddScoped<INotificationRepository>(sp =>
new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
// The NotifyDeliver dispatch path runs through this same long-lived
// provider — a stub adapter that always reports a successful delivery.
centralServices.AddScoped<INotificationDeliveryAdapter>(_ => new AlwaysDeliversAdapter());
await using var centralProvider = centralServices.BuildServiceProvider();
var ingestActor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
(IServiceProvider)centralProvider,
NullLogger<AuditLogIngestActor>.Instance)));
var centralAuditWriter = new CentralAuditWriter(
centralProvider, NullLogger<CentralAuditWriter>.Instance);
// ── Site — SQLite audit writer (hot-path) drained to central by the
// real SiteAuditTelemetryActor through the stub gRPC client. The sync
// ApiCall row and the NotifySend row flow through this chain. ──
await using var sqliteWriter = new SqliteAuditWriter(
Options.Create(new SqliteAuditWriterOptions
{
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 1024,
}),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride:
$"Data Source=file:auditlog-parentexec-{Guid.NewGuid():N}?mode=memory&cache=shared");
var ring = new RingBufferFallback();
var siteAuditWriter = new FallbackAuditWriter(
sqliteWriter, ring, new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
(ISiteAuditQueue)sqliteWriter,
stubClient,
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = 256,
BusyIntervalSeconds = 1,
IdleIntervalSeconds = 1,
}),
NullLogger<SiteAuditTelemetryActor>.Instance)));
// Cached-call telemetry: production forwarder + dispatcher that also
// pushes each combined packet through the stub client into the central
// dual-write transaction (same wiring CombinedTelemetryHarness uses).
var cachedForwarder = new CombinedTelemetryDispatcher(
new CachedCallTelemetryForwarder(
siteAuditWriter, trackingStore: null,
NullLogger<CachedCallTelemetryForwarder>.Instance),
stubClient);
// Site Store-and-Forward — Notify.Send buffers a NotificationSubmit here.
using var safKeepAlive = new Microsoft.Data.Sqlite.SqliteConnection(
$"Data Source=parentexec-saf-{Guid.NewGuid():N};Mode=Memory;Cache=Shared");
safKeepAlive.Open();
var safStorage = new StoreAndForwardStorage(
safKeepAlive.ConnectionString, NullLogger<StoreAndForwardStorage>.Instance);
await safStorage.InitializeAsync();
var storeAndForward = new StoreAndForwardService(
safStorage,
new StoreAndForwardOptions
{
DefaultRetryInterval = TimeSpan.Zero,
DefaultMaxRetries = 3,
RetryTimerInterval = TimeSpan.FromMinutes(10),
},
NullLogger<StoreAndForwardService>.Instance);
// ── Outbound external-system client (routed run): sync Call succeeds,
// CachedCall completes immediately (WasBuffered=false) so the script
// helper emits the Submit + Attempted + CachedResolve lifecycle. ──
var externalClient = Substitute.For<IExternalSystemClient>();
externalClient
.CallAsync(ExternalSystemName, ExternalMethodName,
Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
.Returns(new ExternalCallResult(true, "{\"ok\":true}", null));
externalClient
.CachedCallAsync(ExternalSystemName, ExternalMethodName,
Arg.Any<IReadOnlyDictionary<string, object?>?>(),
Arg.Any<string?>(), Arg.Any<CancellationToken>(),
Arg.Any<ZB.MOM.WW.ScadaBridge.Commons.Types.TrackedOperationId?>(),
Arg.Any<Guid?>(), Arg.Any<string?>(), Arg.Any<Guid?>())
.Returns(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
// ── The routing transport stand-in: builds the routed ScriptRuntimeContext
// carrying RouteToCallRequest.ParentExecutionId — exactly what the
// production site handler (DeploymentManagerActor) does. ──
var router = new BridgingInstanceRouter(
siteId,
externalClient,
siteAuditWriter,
cachedForwarder,
storeAndForward);
// ── The inbound API method script: it calls Route.Call into the site
// instance. The real InboundScriptExecutor binds the inbound request's
// ExecutionId onto the RouteHelper, so the routed call carries it as
// ParentExecutionId. ──
var inboundMethod = new ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi.ApiMethod(
"submitOrder",
$"return await Route.To(\"{RoutedInstanceCode}\").Call(\"{RoutedScriptName}\", new {{ order = 7 }});");
var locator = Substitute.For<IInstanceLocator>();
locator.GetSiteIdForInstanceAsync(RoutedInstanceCode, Arg.Any<CancellationToken>())
.Returns(siteId);
var scriptExecutor = new InboundScriptExecutor(
NullLogger<InboundScriptExecutor>.Instance,
new ServiceCollection().BuildServiceProvider());
Assert.True(scriptExecutor.CompileAndRegister(inboundMethod));
// ── Act — issue the inbound HTTP request through a TestHost pipeline
// fronted by the real AuditWriteMiddleware. The endpoint handler reads
// the middleware-stashed inbound ExecutionId and runs the inbound
// method script with it as parentExecutionId. ──
using var host = await BuildInboundHostAsync(centralAuditWriter, async ctx =>
{
var inboundExecutionId = (Guid)ctx.Items[AuditWriteMiddleware.InboundExecutionIdItemKey]!;
var route = new RouteHelper(locator, router);
var result = await scriptExecutor.ExecuteAsync(
inboundMethod,
new Dictionary<string, object?>(),
route,
TimeSpan.FromSeconds(30),
ctx.RequestAborted,
parentExecutionId: inboundExecutionId);
ctx.Response.StatusCode = result.Success ? 200 : 500;
await ctx.Response.WriteAsync(result.Success ? "ok" : "fail");
});
var client = host.GetTestClient();
var response = await client.PostAsync(
"/api/submitOrder",
new StringContent("{}", Encoding.UTF8, "application/json"));
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
// The routed run emits its sync-ApiCall and NotifySend audit rows on a
// deliberately fire-and-forget path (alog.md §7 — an audit write must
// never block the user-facing script call). `Notify.Send` therefore
// returns — and the routed `RouteToCallAsync` completes — BEFORE the
// SqliteAuditWriter background loop has flushed the NotifySend row into
// the site hot-path. Wait for all five site rows to be durably present
// in SQLite before the central assertion: this is the production
// durability point (the row IS in SQLite before it is considered
// audited), and pinning it removes the emit-vs-drain race that
// otherwise let the SiteAuditTelemetryADrain forward only four rows on
// its first tick and leave NotifySend stranded for a full drain
// interval under heavy parallel load.
await WaitForSiteRowsPersistedAsync(sqliteWriter);
// The routed run produced a NotifySend that buffered a NotificationSubmit
// into S&F. Drain that genuine site-produced submission to the central
// NotificationOutboxActor so the NotifyDeliver dispatch rows materialise.
await ForwardBufferedNotificationToCentralAsync(
storeAndForward, router.NotificationId!, centralProvider, centralAuditWriter);
// ── Assert ──────────────────────────────────────────────────────────
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var repo = new AuditLogRepository(readContext);
// Every audit row this site produced (sync ApiCall + cached lifecycle
// + NotifySend) plus the central NotifyDeliver rows.
var siteRows = await repo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 100));
// sync ApiCall (1) + cached Submit/Attempted/Resolve (3) + NotifySend (1)
// + NotifyDeliver Attempted/Delivered (2) = 7 rows for the routed run.
Assert.True(siteRows.Count == 7,
"Expected 7 routed-run audit rows; saw: "
+ string.Join(", ", siteRows.Select(r => $"{r.Channel}/{r.Kind}/{r.Status}")));
Assert.Single(siteRows, r => r.Channel == AuditChannel.ApiOutbound && r.Kind == AuditKind.ApiCall);
Assert.Single(siteRows, r => r.Kind == AuditKind.CachedSubmit);
Assert.Single(siteRows, r => r.Kind == AuditKind.CachedResolve);
Assert.Single(siteRows, r => r.Kind == AuditKind.NotifySend);
Assert.Equal(2, siteRows.Count(r => r.Kind == AuditKind.NotifyDeliver));
// CORE PROMISE: every routed-run row carries the SAME non-null
// ParentExecutionId — the inbound request's ExecutionId.
var parentIds = siteRows.Select(r => r.ParentExecutionId).Distinct().ToList();
Assert.Single(parentIds);
Assert.NotNull(parentIds[0]);
var inboundExecutionId = parentIds[0]!.Value;
// The routed run has its OWN distinct ExecutionId — not the parent's.
var routedExecutionIds = siteRows
.Select(r => r.ExecutionId)
.Distinct()
.ToList();
Assert.Single(routedExecutionIds);
Assert.NotNull(routedExecutionIds[0]);
var routedExecutionId = routedExecutionIds[0]!.Value;
Assert.NotEqual(inboundExecutionId, routedExecutionId);
// The inbound request's own InboundRequest row is TOP-LEVEL —
// ExecutionId = the propagated id, ParentExecutionId = NULL.
var inboundRows = await repo.QueryAsync(
new AuditLogQueryFilter(ExecutionId: inboundExecutionId),
new AuditLogPaging(PageSize: 10));
var inboundRow = Assert.Single(inboundRows,
r => r.Channel == AuditChannel.ApiInbound && r.Kind == AuditKind.InboundRequest);
Assert.Equal(AuditStatus.Delivered, inboundRow.Status);
Assert.Null(inboundRow.ParentExecutionId);
// The parentExecutionId filter pulls the routed run's complete
// trust-boundary footprint (all 7 routed rows, none of the inbound).
var byParent = await repo.QueryAsync(
new AuditLogQueryFilter(ParentExecutionId: inboundExecutionId),
new AuditLogPaging(PageSize: 100));
Assert.Equal(7, byParent.Count);
Assert.All(byParent, r => Assert.Equal(routedExecutionId, r.ExecutionId));
// GetExecutionTreeAsync returns BOTH executions in one chain —
// inbound (root) and routed (child), regardless of entry point.
var treeFromChild = await repo.GetExecutionTreeAsync(routedExecutionId);
AssertChain(treeFromChild, inboundExecutionId, routedExecutionId);
var treeFromRoot = await repo.GetExecutionTreeAsync(inboundExecutionId);
AssertChain(treeFromRoot, inboundExecutionId, routedExecutionId);
}, TimeSpan.FromSeconds(90));
}
/// <summary>
/// Asserts the execution tree is the expected two-node inbound→routed chain:
/// the inbound execution is the root (<c>ParentExecutionId = NULL</c>) and the
/// routed execution's <c>ParentExecutionId</c> points back at it.
/// </summary>
private static void AssertChain(
IReadOnlyList<ExecutionTreeNode> tree,
Guid inboundExecutionId,
Guid routedExecutionId)
{
Assert.Equal(2, tree.Count);
var root = Assert.Single(tree, n => n.ExecutionId == inboundExecutionId);
Assert.Null(root.ParentExecutionId);
var child = Assert.Single(tree, n => n.ExecutionId == routedExecutionId);
Assert.Equal(inboundExecutionId, child.ParentExecutionId);
}
/// <summary>
/// Spins up a minimal in-memory ASP.NET host whose pipeline mirrors the
/// production inbound-API arrangement: routing → the real
/// <see cref="AuditWriteMiddleware"/> → the <c>POST /api/{methodName}</c>
/// endpoint. The middleware mints + stashes the inbound request's
/// <c>ExecutionId</c> and emits the top-level <see cref="AuditKind.InboundRequest"/>
/// row via the supplied <see cref="ICentralAuditWriter"/>.
/// </summary>
private static async Task<IHost> BuildInboundHostAsync(
ICentralAuditWriter centralAuditWriter,
RequestDelegate endpointHandler)
{
var hostBuilder = new HostBuilder()
.ConfigureWebHost(webBuilder =>
{
webBuilder
.UseTestServer()
.ConfigureServices(services =>
{
services.AddSingleton(centralAuditWriter);
services.AddRouting();
})
.Configure(app =>
{
app.UseRouting();
app.UseAuditWriteMiddleware();
app.UseEndpoints(endpoints =>
{
endpoints.MapPost("/api/{methodName}", endpointHandler);
});
});
});
return await hostBuilder.StartAsync();
}
/// <summary>
/// Reads the genuine site-produced <see cref="NotificationSubmit"/> the routed
/// <c>Notify.Send</c> buffered into Store-and-Forward, then drives it through
/// a real central <see cref="NotificationOutboxActor"/> so the
/// <see cref="AuditKind.NotifyDeliver"/> dispatch rows materialise. The
/// dispatcher echoes <c>OriginParentExecutionId</c> off the
/// <c>NotificationSubmit</c> onto every <c>NotifyDeliver</c> row — the
/// cross-execution linkage under test on the central side.
/// </summary>
private async Task ForwardBufferedNotificationToCentralAsync(
StoreAndForwardService storeAndForward,
string notificationId,
IServiceProvider centralProvider,
ICentralAuditWriter centralAuditWriter)
{
var buffered = await storeAndForward.GetMessageByIdAsync(notificationId);
Assert.NotNull(buffered);
var submit = JsonSerializer.Deserialize<NotificationSubmit>(buffered!.PayloadJson);
Assert.NotNull(submit);
// The routed Notify.Send stamped the inbound request's ExecutionId as the
// submission's OriginParentExecutionId — proven separately on the
// NotifyDeliver rows, but asserted here too as the central handoff input.
Assert.NotNull(submit!.OriginParentExecutionId);
// The outbox actor runs over the long-lived central provider (which
// carries the AlwaysDeliversAdapter) so the dispatch sweep — launched
// asynchronously by the DispatchTick — still has a live IServiceProvider
// to resolve its per-sweep scope from.
var outboxActor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
centralProvider,
new NotificationOutboxOptions
{
// Long timers so PreStart's scheduled ticks never fire — the
// test drives ingest + dispatch explicitly.
DispatchInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromDays(1),
},
centralAuditWriter,
NullLogger<NotificationOutboxActor>.Instance)));
// Ingest the genuine site submission, then run one dispatch sweep.
var ack = await outboxActor.Ask<NotificationSubmitAck>(
submit, TimeSpan.FromSeconds(15));
Assert.True(ack.Accepted, ack.Error);
outboxActor.Tell(InternalMessages.DispatchTick.Instance);
}
/// <summary>
/// Polls the site SQLite hot-path until every audit <see cref="AuditKind"/>
/// the routed run is expected to emit — sync <c>ApiCall</c>, the cached
/// <c>CachedSubmit</c>/<c>ApiCallCached</c>/<c>CachedResolve</c> lifecycle,
/// and <c>NotifySend</c> — is durably present (Pending or Forwarded).
/// </summary>
/// <remarks>
/// The routed run's sync-<c>ApiCall</c> and <c>NotifySend</c> audit rows are
/// written fire-and-forget (the script call must not block on the audit
/// writer — alog.md §7), so the routed <c>RouteToCallAsync</c> returns
/// before the background writer loop has committed those rows.
/// <c>NotifySend</c> is emitted last and therefore settles last. This wait
/// asserts the specific <b>Kinds</b> are present, not merely a row count: a
/// bare count could be satisfied while the last-emitted <c>NotifySend</c>
/// row was still in flight, letting the <c>SiteAuditTelemetryActor</c> drain
/// only a partial snapshot and leave <c>NotifySend</c> stranded for a later
/// tick — the emit-vs-drain race that failed this test under full-suite load.
/// </remarks>
private async Task WaitForSiteRowsPersistedAsync(SqliteAuditWriter sqliteWriter)
{
var expectedKinds = new[]
{
AuditKind.ApiCall, AuditKind.CachedSubmit, AuditKind.ApiCallCached,
AuditKind.CachedResolve, AuditKind.NotifySend,
};
await AwaitAssertAsync(
async () =>
{
var pending = await sqliteWriter.ReadPendingAsync(256);
// AuditLog-001: ReadPendingAsync now excludes the cached-lifecycle
// kinds (they ride the combined-telemetry drain), so we union
// them in via the dedicated read surface to keep the durability
// assertion covering EVERY expected Kind.
var pendingCached = await sqliteWriter.ReadPendingCachedTelemetryAsync(256);
var forwarded = await sqliteWriter.ReadForwardedAsync(256);
var kinds = pending.Concat(pendingCached).Concat(forwarded)
.Select(r => r.Kind).ToHashSet();
var missing = expectedKinds.Where(k => !kinds.Contains(k)).ToList();
Assert.True(
missing.Count == 0,
"Expected every routed-run audit Kind durably in SQLite; missing: "
+ string.Join(", ", missing)
+ $" (saw {pending.Count} Pending + {pendingCached.Count} PendingCached + {forwarded.Count} Forwarded).");
},
TimeSpan.FromSeconds(30),
TimeSpan.FromMilliseconds(50));
}
/// <summary>
/// Stub <see cref="INotificationDeliveryAdapter"/> that always reports a
/// successful delivery — a single dispatch sweep then yields one
/// <see cref="AuditStatus.Attempted"/> + one <see cref="AuditStatus.Delivered"/>
/// <see cref="AuditKind.NotifyDeliver"/> row.
/// </summary>
private sealed class AlwaysDeliversAdapter : INotificationDeliveryAdapter
{
public NotificationType Type => NotificationType.Email;
public Task<DeliveryOutcome> DeliverAsync(
ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications.Notification notification,
CancellationToken cancellationToken = default)
=> Task.FromResult(DeliveryOutcome.Success("ops@example.com"));
}
/// <summary>
/// In-process stand-in for the cross-cluster routing transport
/// (<c>CommunicationServiceInstanceRouter</c> →
/// <c>CommunicationService</c> → site <c>DeploymentManagerActor</c>). On a
/// routed <c>Call</c> it does exactly what the production site handler does:
/// it reads <see cref="RouteToCallRequest.ParentExecutionId"/> off the wire
/// request and threads it into a fresh routed <see cref="ScriptRuntimeContext"/>
/// as <c>parentExecutionId</c>, then runs the routed script's three
/// trust-boundary actions (sync <c>ExternalSystem.Call</c>, a cached call and
/// a <c>Notify.Send</c>). The routed context still mints its OWN fresh
/// <c>ExecutionId</c> — only the parent pointer is inherited.
/// </summary>
private sealed class BridgingInstanceRouter : IInstanceRouter
{
private readonly string _siteId;
private readonly IExternalSystemClient _externalClient;
private readonly IAuditWriter _auditWriter;
private readonly ICachedCallTelemetryForwarder _cachedForwarder;
private readonly StoreAndForwardService _storeAndForward;
/// <summary>
/// The <c>NotificationId</c> the routed <c>Notify.Send</c> minted, captured
/// so the test can drain the buffered <see cref="NotificationSubmit"/>.
/// </summary>
public string? NotificationId { get; private set; }
public BridgingInstanceRouter(
string siteId,
IExternalSystemClient externalClient,
IAuditWriter auditWriter,
ICachedCallTelemetryForwarder cachedForwarder,
StoreAndForwardService storeAndForward)
{
_siteId = siteId;
_externalClient = externalClient;
_auditWriter = auditWriter;
_cachedForwarder = cachedForwarder;
_storeAndForward = storeAndForward;
}
public async Task<RouteToCallResponse> RouteToCallAsync(
string siteId, RouteToCallRequest request, CancellationToken cancellationToken)
{
var compilationService = new ScriptCompilationService(
NullLogger<ScriptCompilationService>.Instance);
var sharedScriptLibrary = new SharedScriptLibrary(
compilationService, NullLogger<SharedScriptLibrary>.Instance);
// Mirror DeploymentManagerActor → ScriptActor → ScriptExecutionActor:
// the routed script execution gets its OWN fresh ExecutionId, and the
// inbound request's ExecutionId arrives as ParentExecutionId.
var routedContext = new ScriptRuntimeContext(
ActorRefs.Nobody,
ActorRefs.Nobody,
sharedScriptLibrary,
currentCallDepth: 0,
maxCallDepth: 10,
askTimeout: TimeSpan.FromSeconds(5),
instanceName: request.InstanceUniqueName,
logger: NullLogger.Instance,
externalSystemClient: _externalClient,
databaseGateway: null,
storeAndForward: _storeAndForward,
siteCommunicationActor: null,
siteId: _siteId,
sourceScript: $"ScriptActor:{request.ScriptName}",
auditWriter: _auditWriter,
operationTrackingStore: null,
cachedForwarder: _cachedForwarder,
executionId: null,
parentExecutionId: request.ParentExecutionId);
// The routed site script's body: a sync ExternalSystem.Call, a cached
// call, and a Notify.Send — three distinct trust-boundary actions of
// the one routed execution.
await routedContext.ExternalSystem.Call(ExternalSystemName, ExternalMethodName);
await routedContext.ExternalSystem.CachedCall(ExternalSystemName, ExternalMethodName);
NotificationId = await routedContext.Notify
.To(NotifyListName)
.Send("Routed run alert", "inbound-routed script fired");
return new RouteToCallResponse(
request.CorrelationId, true, "routed-ok", null, DateTimeOffset.UtcNow);
}
public Task<RouteToGetAttributesResponse> RouteToGetAttributesAsync(
string siteId, RouteToGetAttributesRequest request, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<RouteToSetAttributesResponse> RouteToSetAttributesAsync(
string siteId, RouteToSetAttributesRequest request, CancellationToken cancellationToken)
=> throw new NotSupportedException();
}
}
@@ -0,0 +1,278 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle F (#23 M6-T12) end-to-end tests for the
/// <see cref="AuditLogPartitionMaintenanceService"/> hosted service running
/// the real EF/MSSQL <see cref="AuditLogPartitionMaintenance"/> against the
/// per-class <see cref="MsSqlMigrationFixture"/>. The migration seeds
/// boundaries for every month Jan 2026 Dec 2027, so the eager startup tick
/// can be exercised both for the "future covered" no-op case and for the
/// "lookahead larger than covered" SPLIT case.
/// </summary>
/// <remarks>
/// Tests within this class share one fixture DB — boundaries added by one
/// test persist across the next. Each test reads the max boundary at the
/// start and computes its lookahead relative to it, mirroring the pattern
/// used by the per-component <c>AuditLogPartitionMaintenanceTests</c> in
/// <c>ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests</c>.
/// </remarks>
public class PartitionMaintenanceTests : IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public PartitionMaintenanceTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private ScadaBridgeDbContext CreateContext() =>
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString).Options);
/// <summary>
/// Builds the central-side DI graph for the hosted service: scoped EF
/// context + scoped <see cref="IPartitionMaintenance"/> matching how
/// <c>AddConfigurationDatabase</c> wires the production composition root.
/// </summary>
private ServiceProvider BuildProvider()
{
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(
opts => opts.UseSqlServer(_fixture.ConnectionString),
ServiceLifetime.Scoped);
services.AddScoped<IPartitionMaintenance, AuditLogPartitionMaintenance>();
return services.BuildServiceProvider();
}
private static async Task<DateTime?> ReadMaxBoundaryAsync(IServiceProvider sp)
{
await using var scope = sp.CreateAsyncScope();
var maintenance = scope.ServiceProvider.GetRequiredService<IPartitionMaintenance>();
return await maintenance.GetMaxBoundaryAsync();
}
/// <summary>
/// Mirrors the helper in
/// <c>AuditLogPartitionMaintenanceTests.LookaheadForExtraBoundaries</c>:
/// the smallest lookahead value that lands the SPLIT horizon exactly
/// <paramref name="extraBoundaries"/> months past the current max.
/// </summary>
private static int LookaheadForExtraBoundaries(DateTime max, int extraBoundaries)
{
var nowFirstOfNextMonth = FirstOfNextMonth(DateTime.UtcNow);
var monthsToMax = ((max.Year - nowFirstOfNextMonth.Year) * 12)
+ max.Month - nowFirstOfNextMonth.Month;
return monthsToMax + extraBoundaries;
}
private static int LookaheadInsideExistingRange(DateTime max)
{
var now = DateTime.UtcNow;
var months = ((max.Year - now.Year) * 12) + max.Month - now.Month - 1;
return Math.Max(1, months);
}
private static DateTime FirstOfNextMonth(DateTime instant)
{
var firstOfThisMonth = new DateTime(instant.Year, instant.Month, 1, 0, 0, 0, DateTimeKind.Utc);
return firstOfThisMonth.AddMonths(1);
}
/// <summary>
/// Awaits one full tick of the hosted service. The service runs an
/// eager startup tick inside <see cref="AuditLogPartitionMaintenanceService.StartAsync"/>'s
/// continuation, but the continuation is dispatched on a background
/// Task.Run — so we poll the side effect (the boundary count or
/// max-boundary value) until it changes.
/// </summary>
private async Task StartAndAwaitStartupTickAsync(
AuditLogPartitionMaintenanceService svc,
Func<Task<bool>> awaitCondition,
TimeSpan timeout)
{
await svc.StartAsync(CancellationToken.None);
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (await awaitCondition())
{
return;
}
await Task.Delay(50);
}
}
// ---------------------------------------------------------------------
// 1. EndToEnd_DefaultLookahead_NoSplit_WhenFutureCovered
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_DefaultLookahead_NoSplit_WhenFutureCovered()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await using var sp = BuildProvider();
// The migration seeds boundaries through Dec 2027. With default
// lookahead = 1 and today = ~2026-05-20, horizon =
// NormalizeToFirstOfMonth(now) + 1 = 2026-07-01, well within the
// seeded range, so the startup tick should issue zero SPLITs.
var maxBefore = await ReadMaxBoundaryAsync(sp);
Assert.NotNull(maxBefore);
// Skip if the fixture DB already has boundaries past Dec 2027 from
// a prior test in this class — the lookahead-already-covered path
// is what we want to exercise, regardless of how far past Dec 2027
// the boundary may be.
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
{
IntervalSeconds = 60, // long enough that only the startup tick fires inside the test window
LookaheadMonths = 1,
});
var svc = new AuditLogPartitionMaintenanceService(
sp.GetRequiredService<IServiceScopeFactory>(),
opts,
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
// Drive the startup tick. There is no public completion handle;
// poll until either (a) the max boundary changes (which would be a
// failure for this test) or (b) the polling window expires (success).
await svc.StartAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(2));
await svc.StopAsync(CancellationToken.None);
svc.Dispose();
// Assert the max boundary is unchanged: no SPLIT was issued.
var maxAfter = await ReadMaxBoundaryAsync(sp);
Assert.Equal(maxBefore, maxAfter);
}
// ---------------------------------------------------------------------
// 2. EndToEnd_LookaheadLargerThanCovered_Splits_NewBoundaries
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_LookaheadLargerThanCovered_Splits_NewBoundaries()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await using var sp = BuildProvider();
var maxBefore = await ReadMaxBoundaryAsync(sp);
Assert.NotNull(maxBefore);
// Pick a lookahead that adds exactly two new boundaries past the
// current max. The expected new boundaries are max+1mo and max+2mo.
var lookahead = LookaheadForExtraBoundaries(maxBefore.Value, extraBoundaries: 2);
var expectedFirstNew = maxBefore.Value.AddMonths(1);
var expectedSecondNew = maxBefore.Value.AddMonths(2);
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
{
IntervalSeconds = 60,
LookaheadMonths = lookahead,
});
var svc = new AuditLogPartitionMaintenanceService(
sp.GetRequiredService<IServiceScopeFactory>(),
opts,
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
// Drive the startup tick. Wait until max boundary moves forward by
// the expected amount; SPLIT against MSSQL can take a second or two
// on a busy dev container.
await StartAndAwaitStartupTickAsync(
svc,
async () =>
{
var current = await ReadMaxBoundaryAsync(sp);
return current == expectedSecondNew;
},
timeout: TimeSpan.FromSeconds(15));
await svc.StopAsync(CancellationToken.None);
svc.Dispose();
var maxAfter = await ReadMaxBoundaryAsync(sp);
// Two new boundaries should be present after the startup tick. The
// hosted service does not surface the added-list directly (it logs
// only at Information), so we assert via the max-boundary delta.
Assert.Equal(expectedSecondNew, maxAfter);
// Sanity: the intermediate boundary was also added (the loop
// SPLITs every month from max+1 up to horizon, in order).
Assert.True(expectedFirstNew < expectedSecondNew);
}
// ---------------------------------------------------------------------
// 3. EndToEnd_PartitionMaintenance_Idempotent_OverTwoRuns
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_PartitionMaintenance_Idempotent_OverTwoRuns()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await using var sp = BuildProvider();
var maxBefore = await ReadMaxBoundaryAsync(sp);
Assert.NotNull(maxBefore);
// Add exactly one new boundary on the first run.
var lookahead = LookaheadForExtraBoundaries(maxBefore.Value, extraBoundaries: 1);
var expectedAdded = maxBefore.Value.AddMonths(1);
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
{
IntervalSeconds = 60,
LookaheadMonths = lookahead,
});
// First run.
var svc1 = new AuditLogPartitionMaintenanceService(
sp.GetRequiredService<IServiceScopeFactory>(),
opts,
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
await StartAndAwaitStartupTickAsync(
svc1,
async () =>
{
var current = await ReadMaxBoundaryAsync(sp);
return current == expectedAdded;
},
timeout: TimeSpan.FromSeconds(15));
await svc1.StopAsync(CancellationToken.None);
svc1.Dispose();
var maxAfterFirst = await ReadMaxBoundaryAsync(sp);
Assert.Equal(expectedAdded, maxAfterFirst);
// Second run with the SAME lookahead value. Because the boundary
// is already covered, the EnsureLookaheadAsync call must be a
// no-op — max boundary is unchanged AND no exception is thrown.
var svc2 = new AuditLogPartitionMaintenanceService(
sp.GetRequiredService<IServiceScopeFactory>(),
opts,
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
await svc2.StartAsync(CancellationToken.None);
// Wait long enough that the startup tick would have fired and
// logged any boundary addition; the boundary state must remain
// unchanged after the wait.
await Task.Delay(TimeSpan.FromSeconds(2));
await svc2.StopAsync(CancellationToken.None);
svc2.Dispose();
var maxAfterSecond = await ReadMaxBoundaryAsync(sp);
Assert.Equal(maxAfterFirst, maxAfterSecond);
}
}
@@ -0,0 +1,354 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle F (#23 M6-T11) end-to-end test for the daily partition-switch
/// purge: seeds three monthly partitions (Jan / Feb / Mar 2026) with direct
/// INSERTs that bypass the standard repository ingest path (so the seed
/// timestamps are explicit), drives <see cref="AuditLogPurgeActor"/> against
/// the real <see cref="AuditLogRepository"/> + per-test
/// <see cref="MsSqlMigrationFixture"/> database, and asserts:
/// <list type="number">
/// <item>The oldest partition (Jan) is removed.</item>
/// <item>Newer partitions (Feb + Mar) are untouched.</item>
/// <item>The <c>UX_AuditLog_EventId</c> unique index survives the
/// drop-and-rebuild dance.</item>
/// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains
/// idempotent against the rebuilt index after the purge.</item>
/// </list>
/// </summary>
/// <remarks>
/// The brief calls out that direct INSERTs bypass the writer role's INSERT-only
/// grant; the fixture connects as <c>sa</c> (see
/// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so
/// the seed step does not need the writer role at all. The drop-and-rebuild
/// dance itself runs under the same admin connection because the test owns
/// the database — the role granularity is exercised in the repository tests,
/// not here.
/// </remarks>
public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public PartitionPurgeTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private ScadaBridgeDbContext CreateContext() =>
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString).Options);
/// <summary>
/// Direct INSERT into <c>dbo.AuditLog</c> bypassing
/// <see cref="IAuditLogRepository.InsertIfNotExistsAsync"/>. Used by the
/// seed step so the test can place rows in arbitrary partitions without
/// the repository's idempotency wrapper or ingest-stamping behaviour
/// affecting the seed payload.
/// </summary>
private async Task DirectInsertAsync(
SqlConnection conn,
Guid eventId,
DateTime occurredAtUtc,
string siteId)
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = @"
INSERT INTO dbo.AuditLog
(EventId, OccurredAtUtc, IngestedAtUtc, Channel, Kind, CorrelationId,
SourceSiteId, SourceInstanceId, SourceScript, Actor, Target, Status,
HttpStatus, DurationMs, ErrorMessage, ErrorDetail, RequestSummary,
ResponseSummary, PayloadTruncated, Extra, ForwardState)
VALUES
(@EventId, @OccurredAtUtc, @IngestedAtUtc, 'ApiOutbound', 'ApiCall', NULL,
@SourceSiteId, NULL, NULL, NULL, NULL, 'Delivered',
NULL, NULL, NULL, NULL, NULL,
NULL, 0, NULL, NULL);";
cmd.Parameters.Add("@EventId", System.Data.SqlDbType.UniqueIdentifier).Value = eventId;
// SqlDbType.DateTime2 with explicit Scale 7 matches the
// OccurredAtUtc column shape (datetime2(7)) and avoids the implicit
// narrowing that SqlClient's default DateTime → datetime applies via
// AddWithValue. Critical for partition assignment: the partition
// function key column is datetime2(7); a narrowed value would still
// land in the correct partition for first-of-month seeds, but
// explicit typing here documents the intent and matches how the
// production repository INSERT shapes its parameters.
var occurredParam = cmd.Parameters.Add("@OccurredAtUtc", System.Data.SqlDbType.DateTime2);
occurredParam.Scale = 7;
occurredParam.Value = occurredAtUtc;
var ingestedParam = cmd.Parameters.Add("@IngestedAtUtc", System.Data.SqlDbType.DateTime2);
ingestedParam.Scale = 7;
ingestedParam.Value = DateTime.UtcNow;
cmd.Parameters.Add("@SourceSiteId", System.Data.SqlDbType.VarChar, 64).Value = siteId;
await cmd.ExecuteNonQueryAsync();
}
/// <summary>
/// Asserts that <c>UX_AuditLog_EventId</c> exists in
/// <c>sys.indexes</c>. The drop-and-rebuild dance briefly removes the
/// index inside its transaction; this check is meant to fire AFTER the
/// actor's purge tick has committed so the rebuilt index is observable.
/// </summary>
private static async Task AssertUxIndexExistsAsync(SqlConnection conn)
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = @"
SELECT COUNT(*)
FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId'
AND object_id = OBJECT_ID('dbo.AuditLog');";
var raw = await cmd.ExecuteScalarAsync();
var count = Convert.ToInt32(raw);
Assert.True(count == 1, $"UX_AuditLog_EventId should be present post-purge; sys.indexes count was {count}.");
}
private IActorRef CreateActor(
IServiceProvider sp,
AuditLogPurgeOptions purgeOptions,
AuditLogOptions auditOptions)
{
return Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
sp,
Options.Create(purgeOptions),
Options.Create(auditOptions),
NullLogger<AuditLogPurgeActor>.Instance)));
}
private static (DateTime Jan, DateTime Feb, DateTime Mar) SeedOccurredAt() => (
new DateTime(2026, 1, 15, 0, 0, 0, DateTimeKind.Utc),
new DateTime(2026, 2, 15, 0, 0, 0, DateTimeKind.Utc),
new DateTime(2026, 3, 15, 0, 0, 0, DateTimeKind.Utc));
// ---------------------------------------------------------------------
// 1. EndToEnd_OldestPartition_PurgedViaActor_NewerKept
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_OldestPartition_PurgedViaActor_NewerKept()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Test date is ~2026-05-20 per environment. We want a threshold that
// sits strictly between Jan 15 (the Jan partition's MAX) and Feb 15
// (the Feb partition's MAX) so only the Jan-2026 partition is
// eligible for purge. RetentionDays = 100 gives a threshold of
// ~2026-02-09 — Jan 15 is older (purged), Feb 15 and Mar 15 are
// newer (kept). The window between Jan 15 and Feb 15 is wide enough
// (~30 days) to tolerate any plausible test-clock drift in CI.
var siteId = "purge-e2e-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var janEventId = Guid.NewGuid();
var febEventId = Guid.NewGuid();
var marEventId = Guid.NewGuid();
var (janOccurred, febOccurred, marOccurred) = SeedOccurredAt();
await using (var seedConn = _fixture.OpenConnection())
{
await DirectInsertAsync(seedConn, janEventId, janOccurred, siteId);
await DirectInsertAsync(seedConn, febEventId, febOccurred, siteId);
await DirectInsertAsync(seedConn, marEventId, marOccurred, siteId);
}
// Wire the actor with a real EF context against the fixture DB.
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(
opts => opts.UseSqlServer(_fixture.ConnectionString),
ServiceLifetime.Scoped);
services.AddScoped<IAuditLogRepository, AuditLogRepository>();
var sp = services.BuildServiceProvider();
var probe = CreateTestProbe();
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent));
var purgeOptions = new AuditLogPurgeOptions
{
IntervalHours = 24,
IntervalOverride = TimeSpan.FromMilliseconds(100),
};
var auditOptions = new AuditLogOptions { RetentionDays = 100 };
CreateActor(sp, purgeOptions, auditOptions);
// Wait for the actor's tick to purge the Jan-2026 partition.
// Concurrent test runs against the same fixture might also create
// eligible partitions, but each test class owns its own fixture DB
// (MsSqlMigrationFixture seeds a guid-named DB per class), so the
// Jan-2026 boundary is the only one this test can have produced.
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var matched = probe.FishForMessage<AuditLogPurgedEvent>(
isMessage: m => m.MonthBoundary == janBoundary,
max: TimeSpan.FromSeconds(30));
Assert.True(matched.RowsDeleted >= 1,
$"Expected RowsDeleted >= 1 for Jan-2026 boundary; got {matched.RowsDeleted}.");
// Allow a brief settle in case the actor is mid-tick on Feb/Mar
// (it shouldn't be, since RetentionDays = 90 means only Jan is
// eligible, but the actor MAY re-enumerate quickly while we read).
await Task.Delay(TimeSpan.FromMilliseconds(500));
await using var verify = CreateContext();
var rows = await verify.Set<AuditEvent>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
// Jan removed; Feb + Mar untouched. Because the test owns the site
// id and the fixture DB, exact set membership is observable.
Assert.DoesNotContain(rows, r => r.EventId == janEventId);
Assert.Contains(rows, r => r.EventId == febEventId);
Assert.Contains(rows, r => r.EventId == marEventId);
}
// ---------------------------------------------------------------------
// 2. EndToEnd_UxIndexRebuilt_AfterPurge
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_UxIndexRebuilt_AfterPurge()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Same shape as test 1 — purge the Jan-2026 partition and then
// assert the UX_AuditLog_EventId index is still present. The
// drop-and-rebuild dance briefly removes it inside its transaction
// (the SWITCH PARTITION step requires the non-aligned unique index
// to be absent), but step 5 rebuilds it before committing. Sanity-
// checking the post-COMMIT shape here documents the invariant in an
// assertable way.
var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var janEventId = Guid.NewGuid();
var (janOccurred, _, _) = SeedOccurredAt();
await using (var seedConn = _fixture.OpenConnection())
{
await DirectInsertAsync(seedConn, janEventId, janOccurred, siteId);
}
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(
opts => opts.UseSqlServer(_fixture.ConnectionString),
ServiceLifetime.Scoped);
services.AddScoped<IAuditLogRepository, AuditLogRepository>();
var sp = services.BuildServiceProvider();
var probe = CreateTestProbe();
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent));
CreateActor(
sp,
new AuditLogPurgeOptions
{
IntervalHours = 24,
IntervalOverride = TimeSpan.FromMilliseconds(100),
},
new AuditLogOptions { RetentionDays = 90 });
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
probe.FishForMessage<AuditLogPurgedEvent>(
isMessage: m => m.MonthBoundary == janBoundary,
max: TimeSpan.FromSeconds(30));
// Open a fresh connection (the actor's pool is owned by EF) and
// assert the index is present post-purge.
await using var check = _fixture.OpenConnection();
await AssertUxIndexExistsAsync(check);
}
// ---------------------------------------------------------------------
// 3. EndToEnd_InsertIfNotExistsAsync_StillIdempotent_AfterPurge
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_InsertIfNotExistsAsync_StillIdempotent_AfterPurge()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Seed + purge a Jan-2026 row, THEN exercise InsertIfNotExistsAsync
// twice for a fresh (May-2026) EventId. The second call must be a
// no-op (duplicate-key collision swallowed by the repository, per
// M2 Bundle A's race-fix) — which means the rebuilt
// UX_AuditLog_EventId unique index is functioning as intended.
var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var janEventId = Guid.NewGuid();
var (janOccurred, _, _) = SeedOccurredAt();
await using (var seedConn = _fixture.OpenConnection())
{
await DirectInsertAsync(seedConn, janEventId, janOccurred, siteId);
}
var services = new ServiceCollection();
services.AddDbContext<ScadaBridgeDbContext>(
opts => opts.UseSqlServer(_fixture.ConnectionString),
ServiceLifetime.Scoped);
services.AddScoped<IAuditLogRepository, AuditLogRepository>();
var sp = services.BuildServiceProvider();
var probe = CreateTestProbe();
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent));
CreateActor(
sp,
new AuditLogPurgeOptions
{
IntervalHours = 24,
IntervalOverride = TimeSpan.FromMilliseconds(100),
},
new AuditLogOptions { RetentionDays = 90 });
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
probe.FishForMessage<AuditLogPurgedEvent>(
isMessage: m => m.MonthBoundary == janBoundary,
max: TimeSpan.FromSeconds(30));
// Settle then exercise InsertIfNotExistsAsync twice for the same
// EventId. The repository's idempotency relies on
// UX_AuditLog_EventId being present so the IF NOT EXISTS … INSERT
// race window resolves to a duplicate-key violation the repo
// swallows. If the index were missing here, two rows would land
// and the second InsertIfNotExistsAsync would silently double-insert.
await Task.Delay(TimeSpan.FromMilliseconds(500));
var freshEventId = Guid.NewGuid();
var freshOccurred = new DateTime(2026, 5, 15, 12, 0, 0, DateTimeKind.Utc);
var freshSite = "purge-idem-fresh-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var freshEvt = new AuditEvent
{
EventId = freshEventId,
OccurredAtUtc = freshOccurred,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = freshSite,
Target = "system-x/method",
};
await using (var ctx = CreateContext())
{
var repo = new AuditLogRepository(ctx);
await repo.InsertIfNotExistsAsync(freshEvt);
// Same row a second time — must be a silent no-op.
await repo.InsertIfNotExistsAsync(freshEvt);
}
await using var verify = CreateContext();
var rows = await verify.Set<AuditEvent>()
.Where(e => e.SourceSiteId == freshSite)
.ToListAsync();
Assert.Single(rows);
Assert.Equal(freshEventId, rows[0].EventId);
}
}
@@ -0,0 +1,272 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <summary>
/// Bundle H — end-to-end test wiring the full Audit Log #23 M2 sync-call pipeline:
/// <see cref="FallbackAuditWriter"/> over a <see cref="SqliteAuditWriter"/> backed by
/// an in-memory SQLite database; the <see cref="SiteAuditTelemetryActor"/> drains
/// Pending rows and pushes them through a stub <see cref="ISiteStreamAuditClient"/>
/// that forwards directly to the central <see cref="AuditLogIngestActor"/> backed
/// by a real <see cref="AuditLogRepository"/> on the <see cref="MsSqlMigrationFixture"/>.
/// </summary>
/// <remarks>
/// <para>
/// This is a <b>component-level</b> integration test, not a full Akka-cluster
/// test (per the M2 brainstorm decision). The stub gRPC client short-circuits
/// the wire so we exercise the real telemetry actor, the real ingest actor, the
/// real SQLite writer, and the real MSSQL repository — without standing up a
/// Kestrel host or two-cluster topology.
/// </para>
/// <para>
/// The site-side telemetry actor's <c>Drain</c> message is private; rather than
/// expose it we drive the drain by setting <c>BusyIntervalSeconds = 1</c> so the
/// initial scheduled tick fires within a second of actor start. Tests then
/// <see cref="TestKitBase.AwaitAssertAsync"/> until the central repository
/// observes the expected rows.
/// </para>
/// <para>
/// Each test uses a unique <c>SourceSiteId</c> (Guid suffix) so concurrent tests
/// and the per-fixture MSSQL database lifetime don't interfere with each other.
/// </para>
/// </remarks>
public class SyncCallEmissionEndToEndTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public SyncCallEmissionEndToEndTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private static string NewSiteId() =>
"test-bundle-h-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaBridgeDbContext CreateContext()
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.Options;
return new ScadaBridgeDbContext(options);
}
private static AuditEvent NewEvent(string siteId, Guid? id = null) => new()
{
EventId = id ?? Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = siteId,
Target = "external-system-a/method",
};
private static IOptions<SqliteAuditWriterOptions> InMemorySqliteOptions() =>
Options.Create(new SqliteAuditWriterOptions
{
// Per-test unique database name + Mode=Memory + Cache=Shared keeps
// the in-memory database alive for the duration of the test even
// though Microsoft.Data.Sqlite tears the file down with the last
// connection. The DatabasePath field is unused because we override
// the connection string below.
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 1024,
});
private static SqliteAuditWriter CreateInMemorySqliteWriter() =>
// The 4th constructor argument is connectionStringOverride. A unique
// shared-cache in-memory URI keeps the schema scoped to this writer
// instance and torn down when the writer is disposed.
new SqliteAuditWriter(
InMemorySqliteOptions(),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride: $"Data Source=file:auditlog-h-{Guid.NewGuid():N}?mode=memory&cache=shared");
private static IOptions<SiteAuditTelemetryOptions> FastTelemetryOptions() =>
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = 256,
// 1s for both intervals so the initial scheduled tick fires fast
// and any failure-driven re-tick also fires fast — without
// requiring a public Drain message to be exposed.
BusyIntervalSeconds = 1,
IdleIntervalSeconds = 1,
});
private IActorRef CreateIngestActor(IAuditLogRepository repo) =>
Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
repo,
NullLogger<AuditLogIngestActor>.Instance)));
private IActorRef CreateTelemetryActor(
ISiteAuditQueue queue,
ISiteStreamAuditClient client) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
queue,
client,
FastTelemetryOptions(),
NullLogger<SiteAuditTelemetryActor>.Instance)));
[SkippableFact]
public async Task EndToEnd_OneWrittenEvent_Reaches_Central_AuditLog_Within_Reasonable_Time()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
// Real central wiring: repo + ingest actor.
await using var ingestContext = CreateContext();
var ingestRepo = new AuditLogRepository(ingestContext);
var ingestActor = CreateIngestActor(ingestRepo);
// Real site wiring: SQLite (in-memory) + ring + fallback + telemetry.
await using var sqliteWriter = CreateInMemorySqliteWriter();
var ring = new RingBufferFallback();
var fallback = new FallbackAuditWriter(
sqliteWriter,
ring,
new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
CreateTelemetryActor(sqliteWriter, stubClient);
// Act — one fresh event written via the FallbackAuditWriter hot-path.
var evt = NewEvent(siteId);
await fallback.WriteAsync(evt);
// Assert — the central AuditLog row materialises within a window that
// covers initial tick (1s) + a generous slack for SQLite + the actor
// round-trip + EF/MSSQL latency.
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var readRepo = new AuditLogRepository(readContext);
var rows = await readRepo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 10));
Assert.Single(rows);
Assert.Equal(evt.EventId, rows[0].EventId);
// Central stamps IngestedAtUtc; site never sets it.
Assert.NotNull(rows[0].IngestedAtUtc);
}, TimeSpan.FromSeconds(15));
}
[SkippableFact]
public async Task EndToEnd_GrpcStubError_RowStays_Pending_NextTick_Succeeds()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var ingestContext = CreateContext();
var ingestRepo = new AuditLogRepository(ingestContext);
var ingestActor = CreateIngestActor(ingestRepo);
await using var sqliteWriter = CreateInMemorySqliteWriter();
var ring = new RingBufferFallback();
var fallback = new FallbackAuditWriter(
sqliteWriter,
ring,
new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
// Stub fails the first push; subsequent calls flow through. The
// telemetry actor's on-failure branch keeps rows in Pending state, so
// the next tick re-reads them and tries again.
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor)
{
FailNextCallCount = 1,
};
CreateTelemetryActor(sqliteWriter, stubClient);
var evt = NewEvent(siteId);
await fallback.WriteAsync(evt);
// Wait long enough for at least one failure-then-success cycle. With
// both intervals = 1s the actor retries quickly; allow 15s for slow CI.
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var readRepo = new AuditLogRepository(readContext);
var rows = await readRepo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 10));
Assert.Single(rows);
Assert.Equal(evt.EventId, rows[0].EventId);
}, TimeSpan.FromSeconds(15));
Assert.True(stubClient.CallCount >= 2,
$"Expected at least one failed push + one successful push; saw {stubClient.CallCount} total client calls.");
// The site SQLite row must have flipped to Forwarded after the
// successful retry. ReadPendingAsync only returns Pending rows; the
// row should NOT show up there anymore.
var stillPending = await sqliteWriter.ReadPendingAsync(64);
Assert.DoesNotContain(stillPending, p => p.EventId == evt.EventId);
}
[SkippableFact]
public async Task EndToEnd_DuplicateSubmit_OnlyOneCentralRow()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var ingestContext = CreateContext();
var ingestRepo = new AuditLogRepository(ingestContext);
var ingestActor = CreateIngestActor(ingestRepo);
await using var sqliteWriter = CreateInMemorySqliteWriter();
var ring = new RingBufferFallback();
var fallback = new FallbackAuditWriter(
sqliteWriter,
ring,
new NoOpAuditWriteFailureCounter(),
NullLogger<FallbackAuditWriter>.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
CreateTelemetryActor(sqliteWriter, stubClient);
// Both writes carry the SAME EventId. Site SQLite's PRIMARY KEY
// constraint and the central repo's InsertIfNotExistsAsync both
// enforce first-write-wins, so only one central row must materialise.
var sharedId = Guid.NewGuid();
var evt1 = NewEvent(siteId, sharedId);
var evt2 = NewEvent(siteId, sharedId);
await fallback.WriteAsync(evt1);
await fallback.WriteAsync(evt2);
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var readRepo = new AuditLogRepository(readContext);
var rows = await readRepo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 10));
Assert.Single(rows);
Assert.Equal(sharedId, rows[0].EventId);
}, TimeSpan.FromSeconds(15));
}
}