refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)
Solution + 23 src projects + 26 test projects renamed; folders, csproj, namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated. ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated. SQL roles/logins, LDAP domains, CLI command name, and CLI config dir (~/.scadalink → ~/.scadabridge) also renamed. Build green; 5 Host.Tests fail awaiting SQL login rename in next commit. Pre-existing StaleTagMonitor timing flakes unchanged. Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
+477
@@ -0,0 +1,477 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using System.Text;
|
||||
using System.Text.Encodings.Web;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — M4 Bundle E (Task E4) cross-boundary safety suite verifying
|
||||
/// the alog.md §13 contract: an always-throwing audit writer NEVER aborts the
|
||||
/// user-facing action. Exercises every boundary that emits audit rows in M2,
|
||||
/// M3, and M4:
|
||||
///
|
||||
/// <list type="bullet">
|
||||
/// <item><description>External system sync call (M2 Bundle F).</description></item>
|
||||
/// <item><description>External system cached call (M3 Bundle E).</description></item>
|
||||
/// <item><description>Database sync write (M4 Bundle A).</description></item>
|
||||
/// <item><description>Inbound API request (M4 Bundle D).</description></item>
|
||||
/// <item><description>Notification dispatcher (M4 Bundle B).</description></item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para>
|
||||
/// The site-local boundaries (ESG sync/cached, DB sync) take the always-throw
|
||||
/// <see cref="ThrowingAuditWriter"/> in place of the production
|
||||
/// <see cref="IAuditWriter"/>; the central boundaries (Inbound API,
|
||||
/// Notification dispatcher) take the always-throw
|
||||
/// <see cref="ThrowingCentralAuditWriter"/> in place of
|
||||
/// <see cref="ICentralAuditWriter"/>. In each case the wrapped action's
|
||||
/// original return value (or original exception) must still flow back to the
|
||||
/// caller untouched.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class AuditWriteFailureSafetyTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public AuditWriteFailureSafetyTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Always-throwing writer test doubles
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Site-side <see cref="IAuditWriter"/> that ALWAYS throws on
|
||||
/// <see cref="WriteAsync"/>. Used to verify that ESG / DB script-side
|
||||
/// helpers swallow the throw and return their normal result to the script.
|
||||
/// </summary>
|
||||
private sealed class ThrowingAuditWriter : IAuditWriter
|
||||
{
|
||||
private int _attempts;
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
|
||||
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _attempts);
|
||||
return Task.FromException(new InvalidOperationException(
|
||||
"test-only ThrowingAuditWriter — audit pipeline unavailable"));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Central-side <see cref="ICentralAuditWriter"/> that ALWAYS throws on
|
||||
/// <see cref="WriteAsync"/>. Used to verify Inbound API + Notification
|
||||
/// dispatcher absorb audit-write failures rather than propagating them
|
||||
/// into the response / state transition.
|
||||
/// </summary>
|
||||
private sealed class ThrowingCentralAuditWriter : ICentralAuditWriter
|
||||
{
|
||||
private int _attempts;
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
|
||||
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _attempts);
|
||||
throw new InvalidOperationException(
|
||||
"test-only ThrowingCentralAuditWriter — audit subsystem unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Site-side <see cref="ICachedCallTelemetryForwarder"/> that ALWAYS
|
||||
/// throws on <see cref="ForwardAsync"/>. The cached-call helpers absorb
|
||||
/// the throw and still return a valid <see cref="TrackedOperationId"/>.
|
||||
/// </summary>
|
||||
private sealed class ThrowingCachedForwarder : ICachedCallTelemetryForwarder
|
||||
{
|
||||
private int _attempts;
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
|
||||
public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _attempts);
|
||||
return Task.FromException(new InvalidOperationException(
|
||||
"test-only ThrowingCachedForwarder — telemetry pipeline unavailable"));
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Test 1 — ESG sync call still returns the original ExternalCallResult.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task EsgSyncCall_BrokenAuditWriter_StillReturnsResult()
|
||||
{
|
||||
var client = Substitute.For<IExternalSystemClient>();
|
||||
var expected = new ExternalCallResult(
|
||||
Success: true,
|
||||
ResponseJson: "{\"orderId\":42}",
|
||||
ErrorMessage: null,
|
||||
WasBuffered: false);
|
||||
client.CallAsync(
|
||||
"ERP", "GetOrder",
|
||||
Arg.Any<IReadOnlyDictionary<string, object?>?>(),
|
||||
Arg.Any<CancellationToken>())
|
||||
.Returns(expected);
|
||||
|
||||
var writer = new ThrowingAuditWriter();
|
||||
var helper = new ScriptRuntimeContext.ExternalSystemHelper(
|
||||
client,
|
||||
instanceName: "Plant.Pump42",
|
||||
NullLogger.Instance,
|
||||
Guid.NewGuid(),
|
||||
auditWriter: writer,
|
||||
siteId: "site-77",
|
||||
sourceScript: "ScriptActor:Sync",
|
||||
cachedForwarder: null);
|
||||
|
||||
var result = await helper.Call("ERP", "GetOrder");
|
||||
|
||||
Assert.Same(expected, result);
|
||||
// Proof the audit writer was attempted — otherwise the test wouldn't
|
||||
// actually exercise the safety contract.
|
||||
Assert.True(writer.Attempts >= 1,
|
||||
$"Expected audit writer to be invoked at least once; saw {writer.Attempts}.");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Test 2 — ESG cached call still returns a TrackedOperationId.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task EsgCachedCall_BrokenAuditWriter_StillReturnsTrackedOperationId()
|
||||
{
|
||||
var client = Substitute.For<IExternalSystemClient>();
|
||||
// CachedCallAsync returns WasBuffered=true so the helper takes the
|
||||
// S&F-deferred path — no immediate-terminal telemetry, which keeps the
|
||||
// forwarder attempt count at exactly one (the CachedSubmit emission).
|
||||
client.CachedCallAsync(
|
||||
"ERP", "GetOrder",
|
||||
Arg.Any<IReadOnlyDictionary<string, object?>?>(),
|
||||
Arg.Any<string?>(),
|
||||
Arg.Any<CancellationToken>(),
|
||||
Arg.Any<TrackedOperationId?>())
|
||||
.Returns(new ExternalCallResult(true, null, null, WasBuffered: true));
|
||||
|
||||
// BOTH the audit writer AND the cached forwarder throw — the
|
||||
// CachedSubmit emission goes through the forwarder in production, so
|
||||
// breaking only the writer wouldn't actually exercise the cached
|
||||
// path's safety contract.
|
||||
var writer = new ThrowingAuditWriter();
|
||||
var forwarder = new ThrowingCachedForwarder();
|
||||
var helper = new ScriptRuntimeContext.ExternalSystemHelper(
|
||||
client,
|
||||
instanceName: "Plant.Pump42",
|
||||
NullLogger.Instance,
|
||||
Guid.NewGuid(),
|
||||
auditWriter: writer,
|
||||
siteId: "site-77",
|
||||
sourceScript: "ScriptActor:Cached",
|
||||
cachedForwarder: forwarder);
|
||||
|
||||
var trackedId = await helper.CachedCall("ERP", "GetOrder");
|
||||
|
||||
// Non-default id materialised despite the forwarder failing.
|
||||
Assert.NotEqual(default, trackedId);
|
||||
Assert.True(forwarder.Attempts >= 1,
|
||||
$"Expected cached forwarder to be invoked at least once; saw {forwarder.Attempts}.");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Test 3 — DB sync write still returns the rows-affected count.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task DbSyncWrite_BrokenAuditWriter_StillReturnsRowsAffected()
|
||||
{
|
||||
const string connectionName = "machineData";
|
||||
const string instanceName = "Plant.Pump42";
|
||||
|
||||
using var keepAlive = new SqliteConnection(
|
||||
"Data Source=k-safety-db;Mode=Memory;Cache=Shared");
|
||||
keepAlive.Open();
|
||||
|
||||
// Schema + seed inside a unique in-memory DB.
|
||||
var dbName = $"db-{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
using var dbKeepAlive = new SqliteConnection(connStr);
|
||||
dbKeepAlive.Open();
|
||||
using (var seed = dbKeepAlive.CreateCommand())
|
||||
{
|
||||
seed.CommandText =
|
||||
"CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);";
|
||||
seed.ExecuteNonQuery();
|
||||
}
|
||||
var inner = new SqliteConnection(connStr);
|
||||
inner.Open();
|
||||
|
||||
var gateway = Substitute.For<IDatabaseGateway>();
|
||||
gateway.GetConnectionAsync(connectionName, Arg.Any<CancellationToken>())
|
||||
.Returns(inner);
|
||||
|
||||
var writer = new ThrowingAuditWriter();
|
||||
var helper = new ScriptRuntimeContext.DatabaseHelper(
|
||||
gateway,
|
||||
instanceName,
|
||||
NullLogger.Instance,
|
||||
Guid.NewGuid(),
|
||||
auditWriter: writer,
|
||||
siteId: "site-77",
|
||||
sourceScript: "ScriptActor:Db",
|
||||
cachedForwarder: null);
|
||||
|
||||
await using (var conn = await helper.Connection(connectionName))
|
||||
await using (var cmd = conn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "INSERT INTO t (id, name) VALUES (1, 'safety')";
|
||||
var rows = await cmd.ExecuteNonQueryAsync();
|
||||
Assert.Equal(1, rows);
|
||||
}
|
||||
|
||||
Assert.True(writer.Attempts >= 1,
|
||||
$"Expected audit writer to be invoked at least once; saw {writer.Attempts}.");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Test 4 — Inbound API request still returns HTTP 200.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task InboundApi_BrokenAuditWriter_StillReturns200()
|
||||
{
|
||||
var writer = new ThrowingCentralAuditWriter();
|
||||
|
||||
using var host = await BuildInboundApiHostAsync(writer, endpointStatus: 200);
|
||||
var client = host.GetTestClient();
|
||||
|
||||
var resp = await client.PostAsync(
|
||||
"/api/echo",
|
||||
new StringContent("{\"x\":1}", Encoding.UTF8, "application/json"));
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, resp.StatusCode);
|
||||
Assert.True(writer.Attempts >= 1,
|
||||
$"Expected central audit writer to be invoked at least once; saw {writer.Attempts}.");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Test 5 — Notification dispatcher still transitions to Delivered.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task NotificationDispatch_BrokenAuditWriter_StillTransitionsToDelivered()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = "test-e4-safety-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
var notificationId = Guid.NewGuid();
|
||||
|
||||
await SeedSmtpConfigAsync();
|
||||
await SeedNotificationAsync(notificationId, siteId);
|
||||
|
||||
var adapter = new SingleOutcomeAdapter(DeliveryOutcome.Success("ops@example.com"));
|
||||
var serviceProvider = BuildNotificationDispatcherProvider(adapter);
|
||||
var throwingWriter = new ThrowingCentralAuditWriter();
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
||||
serviceProvider,
|
||||
new NotificationOutboxOptions
|
||||
{
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromDays(1),
|
||||
},
|
||||
(ICentralAuditWriter)throwingWriter,
|
||||
NullLogger<NotificationOutboxActor>.Instance)));
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
// Notifications table reflects the successful delivery even though
|
||||
// every audit write threw — the central direct-write writer
|
||||
// catches/logs internally and the dispatcher catches defensively too
|
||||
// (alog.md §13).
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var row = await ctx.Notifications.SingleAsync(
|
||||
n => n.NotificationId == notificationId.ToString("D"));
|
||||
Assert.Equal(NotificationStatus.Delivered, row.Status);
|
||||
Assert.NotNull(row.DeliveredAt);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
|
||||
Assert.True(throwingWriter.Attempts >= 1,
|
||||
$"Expected dispatcher to attempt audit write at least once; saw {throwingWriter.Attempts}.");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Test infrastructure
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
private IServiceProvider BuildNotificationDispatcherProvider(
|
||||
INotificationDeliveryAdapter adapter)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(opts =>
|
||||
opts.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
|
||||
services.AddScoped<INotificationOutboxRepository>(sp =>
|
||||
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddScoped<INotificationRepository>(sp =>
|
||||
new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private async Task SeedSmtpConfigAsync()
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
// NO-002: dispatcher clamps non-positive RetryDelay to the 1-minute fallback;
|
||||
// use 1 ms so a transient outcome's NextAttemptAt is still effectively due.
|
||||
ctx.SmtpConfigurations.Add(new SmtpConfiguration(
|
||||
"smtp.example.com", "Basic", "noreply@example.com")
|
||||
{
|
||||
MaxRetries = 5,
|
||||
RetryDelay = TimeSpan.FromMilliseconds(1),
|
||||
});
|
||||
await ctx.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task SeedNotificationAsync(Guid notificationId, string siteId)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
ctx.Notifications.Add(new Notification(
|
||||
notificationId.ToString("D"),
|
||||
NotificationType.Email,
|
||||
"ops-team",
|
||||
"Safety subject",
|
||||
"Safety body",
|
||||
siteId)
|
||||
{
|
||||
SourceInstanceId = "Plant.Pump42",
|
||||
SourceScript = "AlarmScript",
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1),
|
||||
});
|
||||
await ctx.SaveChangesAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single-outcome adapter — returns the same <see cref="DeliveryOutcome"/>
|
||||
/// for every call. Used by the dispatcher safety test where we only need
|
||||
/// one happy-path delivery.
|
||||
/// </summary>
|
||||
private sealed class SingleOutcomeAdapter : INotificationDeliveryAdapter
|
||||
{
|
||||
private readonly DeliveryOutcome _outcome;
|
||||
public SingleOutcomeAdapter(DeliveryOutcome outcome) { _outcome = outcome; }
|
||||
public NotificationType Type => NotificationType.Email;
|
||||
public Task<DeliveryOutcome> DeliverAsync(
|
||||
Notification notification, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_outcome);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds an in-memory TestHost mirroring the production inbound-API
|
||||
/// pipeline order. The supplied <paramref name="writer"/> stands in for
|
||||
/// the production <see cref="ICentralAuditWriter"/> so the safety test can
|
||||
/// install the always-throwing variant without standing up any DB.
|
||||
/// </summary>
|
||||
private static async Task<IHost> BuildInboundApiHostAsync(
|
||||
ICentralAuditWriter writer, int endpointStatus)
|
||||
{
|
||||
var hostBuilder = new HostBuilder()
|
||||
.ConfigureWebHost(webBuilder =>
|
||||
{
|
||||
webBuilder
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddSingleton(writer);
|
||||
services.AddRouting();
|
||||
services.AddAuthorization();
|
||||
services.AddAuthentication("TestScheme")
|
||||
.AddScheme<Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions,
|
||||
AlwaysAuthenticatedHandler>("TestScheme", _ => { });
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.UseAuditWriteMiddleware();
|
||||
app.UseEndpoints(endpoints =>
|
||||
{
|
||||
endpoints.MapPost("/api/{methodName}", async ctx =>
|
||||
{
|
||||
ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "safety-actor";
|
||||
ctx.Response.StatusCode = endpointStatus;
|
||||
await ctx.Response.WriteAsync("ok");
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
return await hostBuilder.StartAsync();
|
||||
}
|
||||
|
||||
private sealed class AlwaysAuthenticatedHandler
|
||||
: Microsoft.AspNetCore.Authentication.AuthenticationHandler<
|
||||
Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions>
|
||||
{
|
||||
public AlwaysAuthenticatedHandler(
|
||||
IOptionsMonitor<Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions> options,
|
||||
Microsoft.Extensions.Logging.ILoggerFactory logger,
|
||||
UrlEncoder encoder)
|
||||
: base(options, logger, encoder) { }
|
||||
|
||||
protected override Task<Microsoft.AspNetCore.Authentication.AuthenticateResult>
|
||||
HandleAuthenticateAsync()
|
||||
{
|
||||
var identity = new ClaimsIdentity(
|
||||
new[] { new Claim(ClaimTypes.Name, "framework-user") }, "TestScheme");
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
var ticket = new Microsoft.AspNetCore.Authentication.AuthenticationTicket(
|
||||
principal, "TestScheme");
|
||||
return Task.FromResult(
|
||||
Microsoft.AspNetCore.Authentication.AuthenticateResult.Success(ticket));
|
||||
}
|
||||
}
|
||||
}
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle G G2 end-to-end suite for cached <c>ExternalSystem.CachedCall</c>
|
||||
/// lifecycle telemetry (Audit Log #23 / M3). Wires the full M3 pipeline:
|
||||
/// site-local SQLite audit writer + operation tracking store + the production
|
||||
/// <see cref="CachedCallTelemetryForwarder"/> + the test-side
|
||||
/// <see cref="CombinedTelemetryDispatcher"/> that ALSO pushes each combined
|
||||
/// packet through the stub gRPC client into the central
|
||||
/// <c>AuditLogIngestActor</c>'s dual-write transaction against a per-test
|
||||
/// MSSQL database. Asserts the audit rows + the SiteCalls row + the
|
||||
/// site-local tracking row converge to the expected shape for each lifecycle.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The bridge is driven directly via <see cref="CombinedTelemetryHarness.EmitAttemptAsync"/>
|
||||
/// — these tests do NOT spin up the actual S&F retry loop; that would
|
||||
/// require a full SiteRuntime host and is out of scope for M3 (the S&F
|
||||
/// observer hooks are exercised in <c>ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests</c> at
|
||||
/// unit level). The submit row is emitted via
|
||||
/// <see cref="CombinedTelemetryHarness.EmitSubmitAsync"/> because the
|
||||
/// production submit emission happens at the script-call site, not inside the
|
||||
/// S&F loop.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Each test uses a unique <c>SourceSite</c> id (Guid suffix) so concurrent
|
||||
/// tests sharing the per-fixture MSSQL database don't interfere with each
|
||||
/// other.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class CachedCallCombinedTelemetryTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public CachedCallCombinedTelemetryTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-g2-cached-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private static CachedCallTelemetry SubmitPacket(
|
||||
TrackedOperationId id, string siteId, DateTime nowUtc, string target = "ERP.GetOrder") =>
|
||||
new(
|
||||
Audit: new AuditEvent
|
||||
{
|
||||
EventId = Guid.NewGuid(),
|
||||
OccurredAtUtc = nowUtc,
|
||||
Channel = AuditChannel.ApiOutbound,
|
||||
Kind = AuditKind.CachedSubmit,
|
||||
CorrelationId = id.Value,
|
||||
SourceSiteId = siteId,
|
||||
SourceInstanceId = "Plant.Pump42",
|
||||
SourceScript = "ScriptActor:doStuff",
|
||||
Target = target,
|
||||
Status = AuditStatus.Submitted,
|
||||
ForwardState = AuditForwardState.Pending,
|
||||
},
|
||||
Operational: new SiteCallOperational(
|
||||
TrackedOperationId: id,
|
||||
Channel: "ApiOutbound",
|
||||
Target: target,
|
||||
SourceSite: siteId,
|
||||
SourceNode: null,
|
||||
Status: "Submitted",
|
||||
RetryCount: 0,
|
||||
LastError: null,
|
||||
HttpStatus: null,
|
||||
CreatedAtUtc: nowUtc,
|
||||
UpdatedAtUtc: nowUtc,
|
||||
TerminalAtUtc: null));
|
||||
|
||||
private static CachedCallAttemptContext AttemptContext(
|
||||
TrackedOperationId id,
|
||||
string siteId,
|
||||
CachedCallAttemptOutcome outcome,
|
||||
int retryCount,
|
||||
string? lastError,
|
||||
int? httpStatus,
|
||||
DateTime createdUtc,
|
||||
DateTime occurredUtc,
|
||||
string target = "ERP.GetOrder",
|
||||
string channel = "ApiOutbound") =>
|
||||
new(
|
||||
TrackedOperationId: id,
|
||||
Channel: channel,
|
||||
Target: target,
|
||||
SourceSite: siteId,
|
||||
Outcome: outcome,
|
||||
RetryCount: retryCount,
|
||||
LastError: lastError,
|
||||
HttpStatus: httpStatus,
|
||||
CreatedAtUtc: createdUtc,
|
||||
OccurredAtUtc: occurredUtc,
|
||||
DurationMs: 42,
|
||||
SourceInstanceId: "Plant.Pump42");
|
||||
|
||||
[SkippableFact]
|
||||
public async Task CachedCall_FailFailSuccess_Emits_5_AuditRows_AND_1_SiteCall_Delivered()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var t0 = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
// Submit
|
||||
await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0));
|
||||
|
||||
// Attempt 1: transient HTTP 500
|
||||
await harness.EmitAttemptAsync(AttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.TransientFailure,
|
||||
retryCount: 1, lastError: "HTTP 500", httpStatus: 500,
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(5)));
|
||||
|
||||
// Attempt 2: transient HTTP 500
|
||||
await harness.EmitAttemptAsync(AttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.TransientFailure,
|
||||
retryCount: 2, lastError: "HTTP 500", httpStatus: 500,
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(15)));
|
||||
|
||||
// Attempt 3: delivered (terminal — emits Attempted + CachedResolve)
|
||||
await harness.EmitAttemptAsync(AttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.Delivered,
|
||||
retryCount: 3, lastError: null, httpStatus: 200,
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(25)));
|
||||
|
||||
// Central side: each forward through the dispatcher round-trips
|
||||
// through the stub client + ingest actor, so by the time the awaits
|
||||
// complete the rows are visible in MSSQL.
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
// 1 Submit + 2 transient Attempted + 1 terminal Attempted + 1
|
||||
// CachedResolve = 5 audit rows. The plan allows 4-5; this is the
|
||||
// happy path emitting exactly 5.
|
||||
var auditRows = await read.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
Assert.InRange(auditRows.Count, 4, 5);
|
||||
|
||||
// All audit rows must share the same CorrelationId (= TrackedOperationId).
|
||||
Assert.All(auditRows, r => Assert.Equal(trackedId.Value, r.CorrelationId));
|
||||
|
||||
// Exactly one CachedSubmit row.
|
||||
Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit);
|
||||
// Exactly one terminal CachedResolve row, status Delivered.
|
||||
var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve);
|
||||
Assert.Equal(AuditStatus.Delivered, resolve.Status);
|
||||
|
||||
// SiteCalls row: Delivered, RetryCount=3, TerminalAtUtc set.
|
||||
var siteCall = await read.Set<SiteCall>()
|
||||
.SingleAsync(s => s.TrackedOperationId == trackedId);
|
||||
Assert.Equal("Delivered", siteCall.Status);
|
||||
Assert.Equal(3, siteCall.RetryCount);
|
||||
Assert.NotNull(siteCall.TerminalAtUtc);
|
||||
|
||||
// Site-local Tracking.Status mirrors the same outcome.
|
||||
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
|
||||
Assert.NotNull(snapshot);
|
||||
Assert.Equal("Delivered", snapshot!.Status);
|
||||
Assert.NotNull(snapshot.TerminalAtUtc);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task CachedCall_AllAttemptsFailedAndParked_Emits_Terminal_Parked()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var t0 = new DateTime(2026, 5, 20, 11, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0));
|
||||
|
||||
// Three transient failures...
|
||||
for (int i = 1; i <= 3; i++)
|
||||
{
|
||||
await harness.EmitAttemptAsync(AttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.TransientFailure,
|
||||
retryCount: i, lastError: "HTTP 500", httpStatus: 500,
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(i * 5)));
|
||||
}
|
||||
|
||||
// ...then S&F gives up — ParkedMaxRetries.
|
||||
await harness.EmitAttemptAsync(AttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.ParkedMaxRetries,
|
||||
retryCount: 4, lastError: "HTTP 500", httpStatus: 500,
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(30)));
|
||||
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
var siteCall = await read.Set<SiteCall>()
|
||||
.SingleAsync(s => s.TrackedOperationId == trackedId);
|
||||
Assert.Equal("Parked", siteCall.Status);
|
||||
Assert.NotNull(siteCall.TerminalAtUtc);
|
||||
|
||||
// Terminal audit row should also be Parked.
|
||||
var resolve = await read.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId && e.Kind == AuditKind.CachedResolve)
|
||||
.SingleAsync();
|
||||
Assert.Equal(AuditStatus.Parked, resolve.Status);
|
||||
|
||||
// Site-local tracking matches.
|
||||
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
|
||||
Assert.NotNull(snapshot);
|
||||
Assert.Equal("Parked", snapshot!.Status);
|
||||
Assert.NotNull(snapshot.TerminalAtUtc);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task CachedCall_ImmediateSuccess_NoSF_Emits_Attempted_And_Resolve_Delivered()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
// Submit + immediate delivered attempt (RetryCount = 0).
|
||||
await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0));
|
||||
await harness.EmitAttemptAsync(AttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.Delivered,
|
||||
retryCount: 0, lastError: null, httpStatus: 200,
|
||||
createdUtc: t0, occurredUtc: t0.AddMilliseconds(50)));
|
||||
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
var siteCall = await read.Set<SiteCall>()
|
||||
.SingleAsync(s => s.TrackedOperationId == trackedId);
|
||||
Assert.Equal("Delivered", siteCall.Status);
|
||||
Assert.Equal(0, siteCall.RetryCount);
|
||||
Assert.NotNull(siteCall.TerminalAtUtc);
|
||||
|
||||
// 1 Submit + 1 Attempted + 1 CachedResolve = 3 audit rows.
|
||||
var auditRows = await read.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
Assert.Equal(3, auditRows.Count);
|
||||
Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit);
|
||||
Assert.Single(auditRows, r => r.Kind == AuditKind.ApiCallCached);
|
||||
var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve);
|
||||
Assert.Equal(AuditStatus.Delivered, resolve.Status);
|
||||
|
||||
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
|
||||
Assert.NotNull(snapshot);
|
||||
Assert.Equal("Delivered", snapshot!.Status);
|
||||
}
|
||||
}
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle G G3 mirror of <see cref="CachedCallCombinedTelemetryTests"/> for
|
||||
/// <c>Database.CachedWrite</c>. Same pipeline composition, same dual-write
|
||||
/// transaction, but the lifecycle bridge maps channel <c>"DbOutbound"</c> to
|
||||
/// <see cref="AuditKind.DbWriteCached"/> on the per-attempt audit row (vs.
|
||||
/// <see cref="AuditKind.ApiCallCached"/> for API calls). The
|
||||
/// <see cref="AuditChannel"/> on the audit row, the <c>SiteCalls.Channel</c>
|
||||
/// column, and the per-attempt <see cref="AuditKind"/> all need to come
|
||||
/// through as the DB variants for this path to be considered exercised.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// As with G2, the bridge is driven directly via the harness — we do not
|
||||
/// stand up a real <c>Database.CachedWrite</c> caller. The site-side
|
||||
/// unit-level emission for the DB path is exercised in
|
||||
/// <c>ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests</c>; this suite verifies the end-to-end
|
||||
/// combined-telemetry path produces the right central rows.
|
||||
/// </remarks>
|
||||
public class CachedWriteCombinedTelemetryTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public CachedWriteCombinedTelemetryTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-g3-cachedwrite-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private static CachedCallTelemetry DbSubmitPacket(
|
||||
TrackedOperationId id, string siteId, DateTime nowUtc, string target = "OperationsDb.UpdateOrder") =>
|
||||
new(
|
||||
Audit: new AuditEvent
|
||||
{
|
||||
EventId = Guid.NewGuid(),
|
||||
OccurredAtUtc = nowUtc,
|
||||
Channel = AuditChannel.DbOutbound,
|
||||
Kind = AuditKind.CachedSubmit,
|
||||
CorrelationId = id.Value,
|
||||
SourceSiteId = siteId,
|
||||
SourceInstanceId = "Plant.Pump42",
|
||||
SourceScript = "ScriptActor:doStuff",
|
||||
Target = target,
|
||||
Status = AuditStatus.Submitted,
|
||||
ForwardState = AuditForwardState.Pending,
|
||||
},
|
||||
Operational: new SiteCallOperational(
|
||||
TrackedOperationId: id,
|
||||
Channel: "DbOutbound",
|
||||
Target: target,
|
||||
SourceSite: siteId,
|
||||
SourceNode: null,
|
||||
Status: "Submitted",
|
||||
RetryCount: 0,
|
||||
LastError: null,
|
||||
HttpStatus: null,
|
||||
CreatedAtUtc: nowUtc,
|
||||
UpdatedAtUtc: nowUtc,
|
||||
TerminalAtUtc: null));
|
||||
|
||||
private static CachedCallAttemptContext DbAttemptContext(
|
||||
TrackedOperationId id,
|
||||
string siteId,
|
||||
CachedCallAttemptOutcome outcome,
|
||||
int retryCount,
|
||||
string? lastError,
|
||||
DateTime createdUtc,
|
||||
DateTime occurredUtc,
|
||||
string target = "OperationsDb.UpdateOrder") =>
|
||||
new(
|
||||
TrackedOperationId: id,
|
||||
Channel: "DbOutbound",
|
||||
Target: target,
|
||||
SourceSite: siteId,
|
||||
Outcome: outcome,
|
||||
RetryCount: retryCount,
|
||||
LastError: lastError,
|
||||
HttpStatus: null,
|
||||
CreatedAtUtc: createdUtc,
|
||||
OccurredAtUtc: occurredUtc,
|
||||
DurationMs: 12,
|
||||
SourceInstanceId: "Plant.Pump42");
|
||||
|
||||
[SkippableFact]
|
||||
public async Task CachedWrite_Success_Emits_Delivered_Lifecycle()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var t0 = new DateTime(2026, 5, 20, 13, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
// Submit + immediate delivered attempt.
|
||||
await harness.EmitSubmitAsync(DbSubmitPacket(trackedId, siteId, t0));
|
||||
await harness.EmitAttemptAsync(DbAttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.Delivered,
|
||||
retryCount: 0, lastError: null,
|
||||
createdUtc: t0, occurredUtc: t0.AddMilliseconds(50)));
|
||||
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
// Central SiteCalls row — DbOutbound channel, Delivered.
|
||||
var siteCall = await read.Set<SiteCall>()
|
||||
.SingleAsync(s => s.TrackedOperationId == trackedId);
|
||||
Assert.Equal("DbOutbound", siteCall.Channel);
|
||||
Assert.Equal("Delivered", siteCall.Status);
|
||||
Assert.Equal(0, siteCall.RetryCount);
|
||||
Assert.NotNull(siteCall.TerminalAtUtc);
|
||||
|
||||
var auditRows = await read.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
Assert.Equal(3, auditRows.Count);
|
||||
// Submit row: CachedSubmit + DbOutbound channel.
|
||||
var submit = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit);
|
||||
Assert.Equal(AuditChannel.DbOutbound, submit.Channel);
|
||||
// Per-attempt row: DbWriteCached (NOT ApiCallCached).
|
||||
var attempt = Assert.Single(auditRows, r => r.Kind == AuditKind.DbWriteCached);
|
||||
Assert.Equal(AuditStatus.Attempted, attempt.Status);
|
||||
Assert.Equal(AuditChannel.DbOutbound, attempt.Channel);
|
||||
// Terminal: CachedResolve Delivered.
|
||||
var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve);
|
||||
Assert.Equal(AuditStatus.Delivered, resolve.Status);
|
||||
Assert.Equal(AuditChannel.DbOutbound, resolve.Channel);
|
||||
|
||||
// Site-local tracking row mirrors the same outcome.
|
||||
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
|
||||
Assert.NotNull(snapshot);
|
||||
Assert.Equal("Delivered", snapshot!.Status);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task CachedWrite_Parked_Emits_Terminal_Parked()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var t0 = new DateTime(2026, 5, 20, 14, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
await harness.EmitSubmitAsync(DbSubmitPacket(trackedId, siteId, t0));
|
||||
|
||||
// Two transient SQL-error attempts...
|
||||
for (int i = 1; i <= 2; i++)
|
||||
{
|
||||
await harness.EmitAttemptAsync(DbAttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.TransientFailure,
|
||||
retryCount: i, lastError: "Deadlock victim",
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(i * 5)));
|
||||
}
|
||||
|
||||
// ...then permanent failure → Parked terminal.
|
||||
await harness.EmitAttemptAsync(DbAttemptContext(
|
||||
trackedId, siteId,
|
||||
CachedCallAttemptOutcome.PermanentFailure,
|
||||
retryCount: 3, lastError: "ConstraintViolation: FK_Orders_Customer",
|
||||
createdUtc: t0, occurredUtc: t0.AddSeconds(20)));
|
||||
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
var siteCall = await read.Set<SiteCall>()
|
||||
.SingleAsync(s => s.TrackedOperationId == trackedId);
|
||||
Assert.Equal("DbOutbound", siteCall.Channel);
|
||||
Assert.Equal("Parked", siteCall.Status);
|
||||
Assert.NotNull(siteCall.TerminalAtUtc);
|
||||
|
||||
var resolve = await read.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId && e.Kind == AuditKind.CachedResolve)
|
||||
.SingleAsync();
|
||||
Assert.Equal(AuditStatus.Parked, resolve.Status);
|
||||
Assert.Equal(AuditChannel.DbOutbound, resolve.Channel);
|
||||
|
||||
// Tracking store mirrors Parked.
|
||||
var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId);
|
||||
Assert.NotNull(snapshot);
|
||||
Assert.Equal("Parked", snapshot!.Status);
|
||||
Assert.NotNull(snapshot.TerminalAtUtc);
|
||||
}
|
||||
}
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using Timestamp = Google.Protobuf.WellKnownTypes.Timestamp;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle G G4 idempotency suite. Telemetry packets MUST round-trip safely
|
||||
/// under retried delivery (at-least-once site→central) AND under out-of-order
|
||||
/// arrival (a stale Submit packet arriving after the central row has already
|
||||
/// advanced to Attempted must not regress the SiteCalls status, but must
|
||||
/// still insert its own audit row because audit rows are append-only and the
|
||||
/// lifecycle history is the source of truth for forensics).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pushes <see cref="CachedTelemetryBatch"/> packets directly through the
|
||||
/// stub client (bypassing the local SQLite writer + tracking store) — the
|
||||
/// scenario being modeled is a wire-level retry, not a fresh site call, so
|
||||
/// the local stores' insert/no-op behaviour is already covered by the G2/G3
|
||||
/// happy-path tests. This suite focuses on the central ingest actor's
|
||||
/// dual-write transaction's idempotency contract.
|
||||
/// </remarks>
|
||||
public class CombinedTelemetryIdempotencyTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public CombinedTelemetryIdempotencyTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-g4-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private static CachedTelemetryPacket BuildPacket(
|
||||
Guid eventId,
|
||||
TrackedOperationId trackedId,
|
||||
string siteId,
|
||||
AuditKind kind,
|
||||
AuditStatus auditStatus,
|
||||
string operationalStatus,
|
||||
int retryCount,
|
||||
DateTime nowUtc,
|
||||
DateTime? terminalUtc = null,
|
||||
string? lastError = null,
|
||||
int? httpStatus = null)
|
||||
{
|
||||
var dto = new CachedTelemetryPacket
|
||||
{
|
||||
AuditEvent = AuditEventDtoMapper.ToDto(new AuditEvent
|
||||
{
|
||||
EventId = eventId,
|
||||
OccurredAtUtc = nowUtc,
|
||||
Channel = AuditChannel.ApiOutbound,
|
||||
Kind = kind,
|
||||
CorrelationId = trackedId.Value,
|
||||
SourceSiteId = siteId,
|
||||
Target = "ERP.GetOrder",
|
||||
Status = auditStatus,
|
||||
HttpStatus = httpStatus,
|
||||
ErrorMessage = lastError,
|
||||
ForwardState = AuditForwardState.Pending,
|
||||
}),
|
||||
Operational = new SiteCallOperationalDto
|
||||
{
|
||||
TrackedOperationId = trackedId.Value.ToString("D"),
|
||||
Channel = "ApiOutbound",
|
||||
Target = "ERP.GetOrder",
|
||||
SourceSite = siteId,
|
||||
Status = operationalStatus,
|
||||
RetryCount = retryCount,
|
||||
LastError = lastError ?? string.Empty,
|
||||
CreatedAtUtc = Timestamp.FromDateTime(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc)),
|
||||
UpdatedAtUtc = Timestamp.FromDateTime(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc)),
|
||||
},
|
||||
};
|
||||
if (httpStatus.HasValue)
|
||||
{
|
||||
dto.Operational.HttpStatus = httpStatus.Value;
|
||||
}
|
||||
if (terminalUtc.HasValue)
|
||||
{
|
||||
dto.Operational.TerminalAtUtc =
|
||||
Timestamp.FromDateTime(DateTime.SpecifyKind(terminalUtc.Value, DateTimeKind.Utc));
|
||||
}
|
||||
return dto;
|
||||
}
|
||||
|
||||
private static CachedTelemetryBatch BatchOf(params CachedTelemetryPacket[] packets)
|
||||
{
|
||||
var batch = new CachedTelemetryBatch();
|
||||
batch.Packets.AddRange(packets);
|
||||
return batch;
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task DuplicatePacket_AuditLogStaysAtOneRow_SiteCallsUpserted_Monotonically()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var eventId = Guid.NewGuid();
|
||||
var t0 = new DateTime(2026, 5, 20, 15, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
var packet = BuildPacket(
|
||||
eventId, trackedId, siteId,
|
||||
AuditKind.CachedSubmit, AuditStatus.Submitted, "Submitted",
|
||||
retryCount: 0, nowUtc: t0);
|
||||
|
||||
// First delivery
|
||||
var ack1 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(packet), CancellationToken.None);
|
||||
Assert.Single(ack1.AcceptedEventIds);
|
||||
|
||||
// Second delivery — the exact same packet (simulates a retried gRPC).
|
||||
var ack2 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(packet), CancellationToken.None);
|
||||
// Central acks both deliveries because storage state is consistent —
|
||||
// the site is free to treat its local row as Forwarded either way.
|
||||
Assert.Single(ack2.AcceptedEventIds);
|
||||
Assert.Equal(eventId.ToString(), ack2.AcceptedEventIds[0]);
|
||||
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
// AuditLog: exactly ONE row for the EventId (insert-if-not-exists).
|
||||
var auditCount = await read.Set<AuditEvent>()
|
||||
.CountAsync(e => e.EventId == eventId);
|
||||
Assert.Equal(1, auditCount);
|
||||
|
||||
// SiteCalls: exactly ONE row for the TrackedOperationId.
|
||||
var siteCalls = await read.Set<SiteCall>()
|
||||
.Where(s => s.TrackedOperationId == trackedId)
|
||||
.ToListAsync();
|
||||
Assert.Single(siteCalls);
|
||||
Assert.Equal("Submitted", siteCalls[0].Status);
|
||||
Assert.Equal(0, siteCalls[0].RetryCount);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task OutOfOrderPackets_OlderStatus_ArrivesAfterNewer_IsNoOp()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var trackedId = TrackedOperationId.New();
|
||||
var t0 = new DateTime(2026, 5, 20, 16, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using var harness = new CombinedTelemetryHarness(_fixture, this);
|
||||
|
||||
// First: the Attempted (RetryCount=2) row arrives at central — perhaps
|
||||
// the Submit packet got delayed in flight. SiteCalls advances straight
|
||||
// to Attempted with retry count 2.
|
||||
var attemptedEventId = Guid.NewGuid();
|
||||
var attemptedPacket = BuildPacket(
|
||||
attemptedEventId, trackedId, siteId,
|
||||
AuditKind.ApiCallCached, AuditStatus.Attempted, "Attempted",
|
||||
retryCount: 2, nowUtc: t0.AddSeconds(10),
|
||||
lastError: "HTTP 500", httpStatus: 500);
|
||||
var ack1 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(attemptedPacket), CancellationToken.None);
|
||||
Assert.Single(ack1.AcceptedEventIds);
|
||||
|
||||
// Now the stale Submit packet shows up. The audit row should still be
|
||||
// inserted (audit is append-only — preserve the lifecycle history),
|
||||
// but SiteCalls must NOT regress to Submitted/RetryCount=0.
|
||||
var submitEventId = Guid.NewGuid();
|
||||
var submitPacket = BuildPacket(
|
||||
submitEventId, trackedId, siteId,
|
||||
AuditKind.CachedSubmit, AuditStatus.Submitted, "Submitted",
|
||||
retryCount: 0, nowUtc: t0);
|
||||
var ack2 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(submitPacket), CancellationToken.None);
|
||||
Assert.Single(ack2.AcceptedEventIds);
|
||||
|
||||
await using var read = harness.CreateReadContext();
|
||||
|
||||
// AuditLog: TWO rows now exist for this lifecycle — the Submit and
|
||||
// the Attempted. Their order is by OccurredAtUtc; the test doesn't
|
||||
// assert ordering, only count + correlation.
|
||||
var auditRows = await read.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
Assert.Equal(2, auditRows.Count);
|
||||
Assert.All(auditRows, r => Assert.Equal(trackedId.Value, r.CorrelationId));
|
||||
|
||||
// SiteCalls: stuck at Attempted (monotonic — Submitted is rank 0,
|
||||
// Attempted is rank 2, the upsert for the older row is a no-op).
|
||||
var siteCall = await read.Set<SiteCall>()
|
||||
.SingleAsync(s => s.TrackedOperationId == trackedId);
|
||||
Assert.Equal("Attempted", siteCall.Status);
|
||||
Assert.Equal(2, siteCall.RetryCount);
|
||||
Assert.Equal("HTTP 500", siteCall.LastError);
|
||||
Assert.Equal(500, siteCall.HttpStatus);
|
||||
}
|
||||
}
|
||||
+300
@@ -0,0 +1,300 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — M4 Bundle E (Task E1) end-to-end suite verifying every
|
||||
/// synchronous <c>Database.Connection(name).Execute*</c> /
|
||||
/// <c>ExecuteReader</c> call made via the Bundle A
|
||||
/// <see cref="ScriptRuntimeContext.DatabaseHelper"/> emits exactly one
|
||||
/// <see cref="AuditChannel.DbOutbound"/>/<see cref="AuditKind.DbWrite"/> row
|
||||
/// that materialises in the central MSSQL <c>AuditLog</c> via the production
|
||||
/// site-SQLite + telemetry-actor + central ingest-actor pipeline.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Composes the same pipeline as the M2 <see cref="SyncCallEmissionEndToEndTests"/>:
|
||||
/// in-memory <see cref="SqliteAuditWriter"/> + <see cref="RingBufferFallback"/> +
|
||||
/// <see cref="FallbackAuditWriter"/> on the site, drained by a real
|
||||
/// <see cref="SiteAuditTelemetryActor"/> through a
|
||||
/// <see cref="DirectActorSiteStreamAuditClient"/> stub that short-circuits the
|
||||
/// gRPC wire and Asks the central <see cref="AuditLogIngestActor"/> backed by
|
||||
/// the real <see cref="AuditLogRepository"/> on the per-class
|
||||
/// <see cref="MsSqlMigrationFixture"/> MSSQL database.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Drives the AuditingDbConnection wrapper directly via
|
||||
/// <see cref="ScriptRuntimeContext.DatabaseHelper"/>'s internal ctor (the
|
||||
/// AuditLog tests project has <c>InternalsVisibleTo</c> on SiteRuntime). No
|
||||
/// script runtime, no Akka Instance Actor — the test wires the helper, opens
|
||||
/// an in-memory SQLite connection through a stub <see cref="IDatabaseGateway"/>,
|
||||
/// runs one SQL statement, and waits for the central row to land. Each test
|
||||
/// uses a unique <c>SourceSiteId</c> (Guid suffix) so concurrent tests
|
||||
/// sharing the MSSQL fixture don't interfere with each other.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class DatabaseSyncEmissionEndToEndTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public DatabaseSyncEmissionEndToEndTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private const string ConnectionName = "machineData";
|
||||
private const string InstanceName = "Plant.Pump42";
|
||||
private const string SourceScript = "ScriptActor:doDbWork";
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-e1-db-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-test in-memory SQLite database with a tiny 2-row schema we can both
|
||||
/// write to and select from. Mirrors the pattern from
|
||||
/// <c>DatabaseSyncEmissionTests</c> — the keep-alive root keeps the
|
||||
/// in-memory database file pinned for the duration of the test, while the
|
||||
/// returned <c>live</c> connection is what the stub gateway hands back to
|
||||
/// the auditing wrapper.
|
||||
/// </summary>
|
||||
private static SqliteConnection NewInMemoryDb(out SqliteConnection keepAlive)
|
||||
{
|
||||
var dbName = $"db-{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
|
||||
keepAlive = new SqliteConnection(connStr);
|
||||
keepAlive.Open();
|
||||
using (var seed = keepAlive.CreateCommand())
|
||||
{
|
||||
seed.CommandText =
|
||||
"CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);" +
|
||||
"INSERT INTO t (id, name) VALUES (1, 'alpha');" +
|
||||
"INSERT INTO t (id, name) VALUES (2, 'beta');";
|
||||
seed.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
var live = new SqliteConnection(connStr);
|
||||
live.Open();
|
||||
return live;
|
||||
}
|
||||
|
||||
private static SqliteAuditWriter CreateInMemorySqliteWriter() =>
|
||||
new(
|
||||
Options.Create(new SqliteAuditWriterOptions
|
||||
{
|
||||
DatabasePath = "ignored",
|
||||
BatchSize = 64,
|
||||
ChannelCapacity = 1024,
|
||||
}),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
connectionStringOverride:
|
||||
$"Data Source=file:auditlog-e1-{Guid.NewGuid():N}?mode=memory&cache=shared");
|
||||
|
||||
private static IOptions<SiteAuditTelemetryOptions> FastTelemetryOptions() =>
|
||||
Options.Create(new SiteAuditTelemetryOptions
|
||||
{
|
||||
BatchSize = 256,
|
||||
// 1s on both intervals so the initial scheduled tick fires quickly
|
||||
// — drains the SQLite Pending row and pushes it through the stub
|
||||
// gRPC client into the central ingest actor.
|
||||
BusyIntervalSeconds = 1,
|
||||
IdleIntervalSeconds = 1,
|
||||
});
|
||||
|
||||
private IActorRef CreateIngestActor(IAuditLogRepository repo) =>
|
||||
Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
|
||||
repo,
|
||||
NullLogger<AuditLogIngestActor>.Instance)));
|
||||
|
||||
private IActorRef CreateTelemetryActor(
|
||||
ISiteAuditQueue queue,
|
||||
ISiteStreamAuditClient client) =>
|
||||
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
|
||||
queue,
|
||||
client,
|
||||
FastTelemetryOptions(),
|
||||
NullLogger<SiteAuditTelemetryActor>.Instance)));
|
||||
|
||||
/// <summary>
|
||||
/// Wires the production
|
||||
/// <see cref="ScriptRuntimeContext.DatabaseHelper"/> (internal ctor) onto
|
||||
/// the supplied <see cref="IDatabaseGateway"/> + <see cref="IAuditWriter"/>
|
||||
/// with the test's site id and source script. The returned helper's
|
||||
/// <c>Connection(...)</c> hands back a real <c>AuditingDbConnection</c>.
|
||||
/// </summary>
|
||||
private static ScriptRuntimeContext.DatabaseHelper CreateHelper(
|
||||
IDatabaseGateway gateway,
|
||||
IAuditWriter writer,
|
||||
string siteId) =>
|
||||
new(
|
||||
gateway,
|
||||
InstanceName,
|
||||
NullLogger.Instance,
|
||||
Guid.NewGuid(),
|
||||
auditWriter: writer,
|
||||
siteId: siteId,
|
||||
sourceScript: SourceScript,
|
||||
cachedForwarder: null);
|
||||
|
||||
[SkippableFact]
|
||||
public async Task DbWrite_Insert_Emits_OneCentralRow_WithExtraOpWrite_AndRowsAffected()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
|
||||
// Central — repository + ingest actor backed by the MSSQL fixture.
|
||||
await using var ingestContext = CreateContext();
|
||||
var ingestRepo = new AuditLogRepository(ingestContext);
|
||||
var ingestActor = CreateIngestActor(ingestRepo);
|
||||
|
||||
// Site — SQLite audit writer + ring + fallback + telemetry actor that
|
||||
// drains into the stub gRPC client which forwards to the ingest actor.
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var ring = new RingBufferFallback();
|
||||
var fallback = new FallbackAuditWriter(
|
||||
sqliteWriter,
|
||||
ring,
|
||||
new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
|
||||
CreateTelemetryActor(sqliteWriter, stubClient);
|
||||
|
||||
// SQLite-backed inner connection — the stub gateway hands it to the
|
||||
// auditing wrapper as the DbConnection the script would have got.
|
||||
using var keepAlive = new SqliteConnection("Data Source=k1;Mode=Memory;Cache=Shared");
|
||||
var inner = NewInMemoryDb(out _);
|
||||
var gateway = Substitute.For<IDatabaseGateway>();
|
||||
gateway.GetConnectionAsync(ConnectionName, Arg.Any<CancellationToken>())
|
||||
.Returns(inner);
|
||||
|
||||
// Act — one INSERT through the auditing wrapper. The wrapper emits a
|
||||
// single DbOutbound/DbWrite event to the fallback writer; the
|
||||
// telemetry actor's next tick drains it to central.
|
||||
var helper = CreateHelper(gateway, fallback, siteId);
|
||||
await using (var conn = await helper.Connection(ConnectionName))
|
||||
await using (var cmd = conn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "INSERT INTO t (id, name) VALUES (3, 'gamma')";
|
||||
var rows = await cmd.ExecuteNonQueryAsync();
|
||||
Assert.Equal(1, rows);
|
||||
}
|
||||
|
||||
// Assert — one central row, Kind=DbWrite, Status=Delivered,
|
||||
// Extra.op="write", Extra.rowsAffected=1. 15s upper bound covers the
|
||||
// initial 1s tick + SQLite drain + actor round-trip + EF/MSSQL latency.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var readRepo = new AuditLogRepository(readContext);
|
||||
var rows = await readRepo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
var evt = Assert.Single(rows);
|
||||
Assert.Equal(AuditChannel.DbOutbound, evt.Channel);
|
||||
Assert.Equal(AuditKind.DbWrite, evt.Kind);
|
||||
Assert.Equal(AuditStatus.Delivered, evt.Status);
|
||||
Assert.Equal(siteId, evt.SourceSiteId);
|
||||
Assert.Equal(InstanceName, evt.SourceInstanceId);
|
||||
Assert.Equal(SourceScript, evt.SourceScript);
|
||||
Assert.NotNull(evt.Extra);
|
||||
Assert.Contains("\"op\":\"write\"", evt.Extra);
|
||||
Assert.Contains("\"rowsAffected\":1", evt.Extra);
|
||||
// Central stamps IngestedAtUtc; the site never sets it.
|
||||
Assert.NotNull(evt.IngestedAtUtc);
|
||||
Assert.StartsWith(ConnectionName, evt.Target);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task DbWrite_Select_Emits_OneCentralRow_WithExtraOpRead_AndRowsReturned()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
|
||||
await using var ingestContext = CreateContext();
|
||||
var ingestRepo = new AuditLogRepository(ingestContext);
|
||||
var ingestActor = CreateIngestActor(ingestRepo);
|
||||
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var ring = new RingBufferFallback();
|
||||
var fallback = new FallbackAuditWriter(
|
||||
sqliteWriter,
|
||||
ring,
|
||||
new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
|
||||
CreateTelemetryActor(sqliteWriter, stubClient);
|
||||
|
||||
using var keepAlive = new SqliteConnection("Data Source=k2;Mode=Memory;Cache=Shared");
|
||||
var inner = NewInMemoryDb(out _);
|
||||
var gateway = Substitute.For<IDatabaseGateway>();
|
||||
gateway.GetConnectionAsync(ConnectionName, Arg.Any<CancellationToken>())
|
||||
.Returns(inner);
|
||||
|
||||
var helper = CreateHelper(gateway, fallback, siteId);
|
||||
await using (var conn = await helper.Connection(ConnectionName))
|
||||
await using (var cmd = conn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT id, name FROM t ORDER BY id";
|
||||
await using var reader = await cmd.ExecuteReaderAsync();
|
||||
var seen = 0;
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
seen++;
|
||||
}
|
||||
// Explicit close so the AuditingDbDataReader callback fires before
|
||||
// the helper is disposed (Bundle A defers the audit emission to
|
||||
// reader-close so rowsReturned is observable).
|
||||
await reader.CloseAsync();
|
||||
Assert.Equal(2, seen);
|
||||
}
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var readRepo = new AuditLogRepository(readContext);
|
||||
var rows = await readRepo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
var evt = Assert.Single(rows);
|
||||
Assert.Equal(AuditChannel.DbOutbound, evt.Channel);
|
||||
Assert.Equal(AuditKind.DbWrite, evt.Kind);
|
||||
Assert.Equal(AuditStatus.Delivered, evt.Status);
|
||||
Assert.NotNull(evt.Extra);
|
||||
Assert.Contains("\"op\":\"read\"", evt.Extra);
|
||||
Assert.Contains("\"rowsReturned\":2", evt.Extra);
|
||||
Assert.NotNull(evt.IngestedAtUtc);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — ExecutionId end-to-end correlation suite verifying the
|
||||
/// universal per-run correlation promise: <b>every audit row produced by one
|
||||
/// script execution carries the same non-null <see cref="AuditEvent.ExecutionId"/></b>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is the integration-level counterpart to the unit-level
|
||||
/// <c>ExecutionCorrelationContextTests</c> in <c>ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests</c>:
|
||||
/// where that test asserts the shared id on the in-memory captured rows, this
|
||||
/// suite drives the rows all the way through the production pipeline — the real
|
||||
/// <see cref="SqliteAuditWriter"/> site hot-path, the real
|
||||
/// <see cref="SiteAuditTelemetryActor"/> drain loop, the real
|
||||
/// <see cref="AuditLogIngestActor"/>, and the real <see cref="AuditLogRepository"/>
|
||||
/// over the per-class <see cref="MsSqlMigrationFixture"/> MSSQL database — then
|
||||
/// reads the rows back from the central store and asserts the shared id.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Composes the same pipeline as the M2 <see cref="SyncCallEmissionEndToEndTests"/>
|
||||
/// and the M4 <see cref="DatabaseSyncEmissionEndToEndTests"/>: an in-memory
|
||||
/// <see cref="SqliteAuditWriter"/> + <see cref="RingBufferFallback"/> +
|
||||
/// <see cref="FallbackAuditWriter"/> on the site, drained by a real
|
||||
/// <see cref="SiteAuditTelemetryActor"/> through the shared
|
||||
/// <see cref="DirectActorSiteStreamAuditClient"/> stub that short-circuits the
|
||||
/// gRPC wire and Asks the central ingest actor. The production
|
||||
/// <see cref="ScriptRuntimeContext"/> is driven directly: one context performs
|
||||
/// two distinct trust-boundary actions — a sync <c>ExternalSystem.Call</c> and a
|
||||
/// sync <c>Database</c> write — so the two emitted audit rows originate from one
|
||||
/// execution. Each test uses a unique <c>ExecutionId</c> + <c>SourceSiteId</c>
|
||||
/// (Guid suffixes) so concurrent tests sharing the MSSQL fixture don't interfere.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class ExecutionIdCorrelationTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public ExecutionIdCorrelationTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private const string ConnectionName = "machineData";
|
||||
private const string InstanceName = "Plant.Pump42";
|
||||
private const string SourceScript = "ScriptActor:OnTick";
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-execid-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-test in-memory SQLite database with a tiny single-table schema the
|
||||
/// sync DB write targets. The keep-alive root pins the in-memory file for
|
||||
/// the duration of the test; the returned <c>live</c> connection is what the
|
||||
/// stub gateway hands back to the auditing wrapper. Mirrors
|
||||
/// <c>DatabaseSyncEmissionEndToEndTests.NewInMemoryDb</c>.
|
||||
/// </summary>
|
||||
private static SqliteConnection NewInMemoryDb(out SqliteConnection keepAlive)
|
||||
{
|
||||
var dbName = $"db-{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
|
||||
keepAlive = new SqliteConnection(connStr);
|
||||
keepAlive.Open();
|
||||
using (var seed = keepAlive.CreateCommand())
|
||||
{
|
||||
seed.CommandText =
|
||||
"CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);";
|
||||
seed.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
var live = new SqliteConnection(connStr);
|
||||
live.Open();
|
||||
return live;
|
||||
}
|
||||
|
||||
private static SqliteAuditWriter CreateInMemorySqliteWriter() =>
|
||||
new(
|
||||
Options.Create(new SqliteAuditWriterOptions
|
||||
{
|
||||
DatabasePath = "ignored",
|
||||
BatchSize = 64,
|
||||
ChannelCapacity = 1024,
|
||||
}),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
connectionStringOverride:
|
||||
$"Data Source=file:auditlog-execid-{Guid.NewGuid():N}?mode=memory&cache=shared");
|
||||
|
||||
private static IOptions<SiteAuditTelemetryOptions> FastTelemetryOptions() =>
|
||||
Options.Create(new SiteAuditTelemetryOptions
|
||||
{
|
||||
BatchSize = 256,
|
||||
// 1s on both intervals so the initial scheduled tick fires quickly
|
||||
// — drains the SQLite Pending rows and pushes them through the stub
|
||||
// gRPC client into the central ingest actor.
|
||||
BusyIntervalSeconds = 1,
|
||||
IdleIntervalSeconds = 1,
|
||||
});
|
||||
|
||||
private IActorRef CreateIngestActor(IAuditLogRepository repo) =>
|
||||
Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
|
||||
repo,
|
||||
NullLogger<AuditLogIngestActor>.Instance)));
|
||||
|
||||
private IActorRef CreateTelemetryActor(
|
||||
ISiteAuditQueue queue,
|
||||
ISiteStreamAuditClient client) =>
|
||||
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
|
||||
queue,
|
||||
client,
|
||||
FastTelemetryOptions(),
|
||||
NullLogger<SiteAuditTelemetryActor>.Instance)));
|
||||
|
||||
[SkippableFact]
|
||||
public async Task OneExecution_ApiCallAndDbWrite_AllCentralRows_ShareOneNonNullExecutionId()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
// An explicit per-run execution id — the value the test asserts on every
|
||||
// audit row produced by the single script execution below.
|
||||
var executionId = Guid.NewGuid();
|
||||
|
||||
// ── Central — repository + ingest actor backed by the MSSQL fixture ──
|
||||
await using var ingestContext = CreateContext();
|
||||
var ingestRepo = new AuditLogRepository(ingestContext);
|
||||
var ingestActor = CreateIngestActor(ingestRepo);
|
||||
|
||||
// ── Site — SQLite audit writer + ring + fallback + telemetry actor ───
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var ring = new RingBufferFallback();
|
||||
var fallback = new FallbackAuditWriter(
|
||||
sqliteWriter,
|
||||
ring,
|
||||
new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
|
||||
CreateTelemetryActor(sqliteWriter, stubClient);
|
||||
|
||||
// Outbound API client — one successful CallAsync, one audit row.
|
||||
var externalClient = Substitute.For<IExternalSystemClient>();
|
||||
externalClient
|
||||
.CallAsync("ERP", "GetOrder", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ExternalCallResult(true, "{}", null));
|
||||
|
||||
// SQLite-backed inner DB connection — the stub gateway hands it to the
|
||||
// auditing wrapper as the connection the script would have got.
|
||||
using var keepAlive = new SqliteConnection("Data Source=execid-k1;Mode=Memory;Cache=Shared");
|
||||
var innerDb = NewInMemoryDb(out _);
|
||||
var gateway = Substitute.For<IDatabaseGateway>();
|
||||
gateway.GetConnectionAsync(ConnectionName, Arg.Any<CancellationToken>())
|
||||
.Returns(innerDb);
|
||||
|
||||
// ── Act — ONE script execution: a sync ExternalSystem.Call AND a sync
|
||||
// Database write, both performed through a SINGLE ScriptRuntimeContext
|
||||
// stamped with the explicit executionId. Each helper emits exactly one
|
||||
// trust-boundary audit row to the fallback writer; the telemetry actor's
|
||||
// next tick drains both to central.
|
||||
var context = CreateScriptContext(externalClient, gateway, fallback, siteId, executionId);
|
||||
|
||||
await context.ExternalSystem.Call("ERP", "GetOrder");
|
||||
|
||||
await using (var conn = await context.Database.Connection(ConnectionName))
|
||||
await using (var cmd = conn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "INSERT INTO t (id, name) VALUES (1, 'alpha')";
|
||||
var affected = await cmd.ExecuteNonQueryAsync();
|
||||
Assert.Equal(1, affected);
|
||||
}
|
||||
|
||||
// ── Assert — read the rows back from the CENTRAL store filtered by the
|
||||
// execution id; both the ApiCall and the DbWrite row must be present and
|
||||
// every one must carry the SAME non-null ExecutionId we minted above.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var readRepo = new AuditLogRepository(readContext);
|
||||
|
||||
// The ExecutionId filter dimension is the universal-correlation
|
||||
// query an audit reader uses to pull every action of one run.
|
||||
var rows = await readRepo.QueryAsync(
|
||||
new AuditLogQueryFilter(ExecutionId: executionId),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
|
||||
// Both trust-boundary actions of the one execution have landed.
|
||||
Assert.Equal(2, rows.Count);
|
||||
|
||||
// Every central row carries the SAME non-null ExecutionId — the
|
||||
// core promise of the per-run correlation value.
|
||||
Assert.All(rows, r =>
|
||||
{
|
||||
Assert.NotNull(r.ExecutionId);
|
||||
Assert.Equal(executionId, r.ExecutionId);
|
||||
Assert.Equal(siteId, r.SourceSiteId);
|
||||
// Central stamps IngestedAtUtc; the site never sets it.
|
||||
Assert.NotNull(r.IngestedAtUtc);
|
||||
});
|
||||
|
||||
// The two rows are the two distinct trust-boundary actions — one
|
||||
// outbound API call and one outbound DB write — proving the shared
|
||||
// id spans different channels, not two rows of the same action.
|
||||
Assert.Single(rows, r => r.Channel == AuditChannel.ApiOutbound && r.Kind == AuditKind.ApiCall);
|
||||
Assert.Single(rows, r => r.Channel == AuditChannel.DbOutbound && r.Kind == AuditKind.DbWrite);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a production <see cref="ScriptRuntimeContext"/> wired with the
|
||||
/// outbound external-system client, the database gateway and the audit
|
||||
/// writer, stamped with an explicit <paramref name="executionId"/>. The
|
||||
/// actor refs are <see cref="ActorRefs.Nobody"/> — the ExternalSystem /
|
||||
/// Database helpers exercised here never touch them.
|
||||
/// </summary>
|
||||
private static ScriptRuntimeContext CreateScriptContext(
|
||||
IExternalSystemClient externalSystemClient,
|
||||
IDatabaseGateway databaseGateway,
|
||||
IAuditWriter auditWriter,
|
||||
string siteId,
|
||||
Guid executionId)
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
var sharedScriptLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
|
||||
return new ScriptRuntimeContext(
|
||||
ActorRefs.Nobody,
|
||||
ActorRefs.Nobody,
|
||||
sharedScriptLibrary,
|
||||
currentCallDepth: 0,
|
||||
maxCallDepth: 10,
|
||||
askTimeout: TimeSpan.FromSeconds(5),
|
||||
instanceName: InstanceName,
|
||||
logger: NullLogger.Instance,
|
||||
externalSystemClient: externalSystemClient,
|
||||
databaseGateway: databaseGateway,
|
||||
storeAndForward: null,
|
||||
siteCommunicationActor: null,
|
||||
siteId: siteId,
|
||||
sourceScript: SourceScript,
|
||||
auditWriter: auditWriter,
|
||||
operationTrackingStore: null,
|
||||
cachedForwarder: null,
|
||||
executionId: executionId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
|
||||
using System.Security.Claims;
|
||||
using System.Text;
|
||||
using System.Text.Encodings.Web;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — M4 Bundle E (Task E3) end-to-end audit trail for the
|
||||
/// inbound API surface. Wires the production
|
||||
/// <see cref="AuditWriteMiddleware"/> into a Microsoft.AspNetCore.TestHost
|
||||
/// pipeline that mirrors the production
|
||||
/// <c>UseAuthentication → UseAuditWriteMiddleware → POST /api/{methodName}</c>
|
||||
/// order, with the real <see cref="CentralAuditWriter"/> backed by the per-class
|
||||
/// <see cref="MsSqlMigrationFixture"/> MSSQL <c>AuditLog</c> table.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Three response shapes are covered: a happy-path 200 (with the actor
|
||||
/// resolved from <see cref="HttpContext.Items"/>), a 401 unauthenticated
|
||||
/// (Actor stays null, kind flips to
|
||||
/// <see cref="AuditKind.InboundAuthFailure"/>), and a 500 internal-error
|
||||
/// response. Each test uses a unique method name so concurrent tests sharing
|
||||
/// the fixture don't interfere.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The middleware-level unit tests already cover the recording-writer shape
|
||||
/// (<c>AuditWriteMiddlewareTests</c>) and the pipeline ordering
|
||||
/// (<c>MiddlewareOrderTests</c>); these tests verify the END-TO-END
|
||||
/// materialisation in the central <c>AuditLog</c> table — the production
|
||||
/// glue from request → writer → repository → MSSQL row.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class InboundApiAuditTests : IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public InboundApiAuditTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-test unique method name suffix — the audit row's <c>Target</c>
|
||||
/// captures it so each test can query by <see cref="AuditLogQueryFilter.Target"/>
|
||||
/// without disturbing other tests using the same MSSQL fixture.
|
||||
/// </summary>
|
||||
private static string NewMethodName(string prefix) =>
|
||||
prefix + "-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Spins up a minimal in-memory ASP.NET host whose pipeline mirrors the
|
||||
/// production arrangement. The endpoint handler delegate is supplied by
|
||||
/// each test so it can shape the response (200 with an actor, 401
|
||||
/// auth-fail, 500 server error) the way the production handler would.
|
||||
/// </summary>
|
||||
private async Task<IHost> BuildHostAsync(RequestDelegate endpointHandler)
|
||||
{
|
||||
var hostBuilder = new HostBuilder()
|
||||
.ConfigureWebHost(webBuilder =>
|
||||
{
|
||||
webBuilder
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
// Real EF DbContext + AuditLogRepository wired against
|
||||
// the per-class MSSQL fixture. CentralAuditWriter is a
|
||||
// singleton — same pattern the production Host uses —
|
||||
// opening a fresh scope per call to resolve the scoped
|
||||
// repository.
|
||||
services.AddDbContext<ScadaBridgeDbContext>(opts =>
|
||||
opts.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
|
||||
services.AddScoped<IAuditLogRepository>(sp =>
|
||||
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddSingleton<ICentralAuditWriter>(sp =>
|
||||
new CentralAuditWriter(sp, NullLogger<CentralAuditWriter>.Instance));
|
||||
|
||||
services.AddRouting();
|
||||
services.AddAuthorization();
|
||||
services.AddAuthentication("TestScheme")
|
||||
.AddScheme<AuthenticationSchemeOptions, AlwaysAuthenticatedHandler>(
|
||||
"TestScheme", _ => { });
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
// Mirror production order: routing → auth → audit
|
||||
// middleware → endpoint. The auth scheme always
|
||||
// succeeds; per-request auth-failure semantics are
|
||||
// produced INSIDE the endpoint handler (mirroring
|
||||
// ApiKeyValidator's in-handler short-circuit).
|
||||
app.UseRouting();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.UseAuditWriteMiddleware();
|
||||
app.UseEndpoints(endpoints =>
|
||||
{
|
||||
endpoints.MapPost("/api/{methodName}", endpointHandler);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
return await hostBuilder.StartAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal authentication handler that always succeeds — keeps
|
||||
/// <see cref="HttpContext.User"/> populated so the middleware's
|
||||
/// Items-then-User fallback path has a real principal to ignore. The
|
||||
/// middleware's primary actor resolution path uses
|
||||
/// <see cref="AuditWriteMiddleware.AuditActorItemKey"/> so this handler's
|
||||
/// claim never appears on the emitted Actor unless the endpoint stashes
|
||||
/// it explicitly.
|
||||
/// </summary>
|
||||
private sealed class AlwaysAuthenticatedHandler : AuthenticationHandler<AuthenticationSchemeOptions>
|
||||
{
|
||||
public AlwaysAuthenticatedHandler(
|
||||
Microsoft.Extensions.Options.IOptionsMonitor<AuthenticationSchemeOptions> options,
|
||||
Microsoft.Extensions.Logging.ILoggerFactory logger,
|
||||
UrlEncoder encoder)
|
||||
: base(options, logger, encoder) { }
|
||||
|
||||
protected override Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||
{
|
||||
var identity = new ClaimsIdentity(
|
||||
new[] { new Claim(ClaimTypes.Name, "framework-user") }, "TestScheme");
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
var ticket = new AuthenticationTicket(principal, "TestScheme");
|
||||
return Task.FromResult(AuthenticateResult.Success(ticket));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queries the central <c>AuditLog</c> table for the row produced by the
|
||||
/// test's unique method name. Wrapped in <see cref="Assert"/> calls so the
|
||||
/// query can be used inside a polling helper.
|
||||
/// </summary>
|
||||
private async Task<IReadOnlyList<AuditEvent>> QueryByTargetAsync(string methodName)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
return await repo.QueryAsync(
|
||||
new AuditLogQueryFilter(Target: methodName),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Awaits the central <c>AuditLog</c> row materialising for
|
||||
/// <paramref name="methodName"/>. The writer is fire-and-forget so we
|
||||
/// poll briefly after the HTTP response returns to absorb scheduling
|
||||
/// jitter between the middleware's <c>finally</c> block and the row hitting
|
||||
/// MSSQL.
|
||||
/// </summary>
|
||||
private async Task<AuditEvent> AwaitOneAsync(string methodName)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var rows = await QueryByTargetAsync(methodName);
|
||||
if (rows.Count > 0)
|
||||
{
|
||||
return Assert.Single(rows);
|
||||
}
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
// Fall through to a final query so the failure message carries the
|
||||
// actual row count from the last attempt.
|
||||
var finalRows = await QueryByTargetAsync(methodName);
|
||||
return Assert.Single(finalRows);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task PostToApi_WithValidActor_Emits_InboundRequest_StatusDelivered_HttpStatus200_ActorPopulated()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var methodName = NewMethodName("echo");
|
||||
|
||||
using var host = await BuildHostAsync(async ctx =>
|
||||
{
|
||||
// Simulate the production endpoint stashing the resolved API key
|
||||
// name on HttpContext.Items AFTER successful auth — the middleware
|
||||
// reads it in its finally block to populate Actor.
|
||||
ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "integration-svc";
|
||||
ctx.Response.StatusCode = 200;
|
||||
await ctx.Response.WriteAsync("ok");
|
||||
});
|
||||
|
||||
var client = host.GetTestClient();
|
||||
var resp = await client.PostAsync(
|
||||
$"/api/{methodName}",
|
||||
new StringContent("{\"x\":1}", Encoding.UTF8, "application/json"));
|
||||
Assert.Equal(System.Net.HttpStatusCode.OK, resp.StatusCode);
|
||||
|
||||
var evt = await AwaitOneAsync(methodName);
|
||||
Assert.Equal(AuditChannel.ApiInbound, evt.Channel);
|
||||
Assert.Equal(AuditKind.InboundRequest, evt.Kind);
|
||||
Assert.Equal(AuditStatus.Delivered, evt.Status);
|
||||
Assert.Equal(200, evt.HttpStatus);
|
||||
Assert.Equal("integration-svc", evt.Actor);
|
||||
// Central direct-write — no site-local forward state (alog.md §6).
|
||||
Assert.Null(evt.ForwardState);
|
||||
// IngestedAtUtc stamped by the central writer.
|
||||
Assert.NotNull(evt.IngestedAtUtc);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task PostToApi_Without_Auth_Emits_InboundAuthFailure_StatusFailed_HttpStatus401_ActorNull()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var methodName = NewMethodName("auth-fail");
|
||||
|
||||
using var host = await BuildHostAsync(async ctx =>
|
||||
{
|
||||
// The production ApiKeyValidator returns 401 from inside the
|
||||
// handler when the X-API-Key header is missing or invalid; the
|
||||
// handler must NOT stash an actor name in that case so the
|
||||
// middleware emits Actor=null on the resulting audit row.
|
||||
ctx.Response.StatusCode = 401;
|
||||
await ctx.Response.WriteAsync("unauthorized");
|
||||
});
|
||||
|
||||
var client = host.GetTestClient();
|
||||
var resp = await client.PostAsync(
|
||||
$"/api/{methodName}",
|
||||
new StringContent("{}", Encoding.UTF8, "application/json"));
|
||||
Assert.Equal(System.Net.HttpStatusCode.Unauthorized, resp.StatusCode);
|
||||
|
||||
var evt = await AwaitOneAsync(methodName);
|
||||
Assert.Equal(AuditChannel.ApiInbound, evt.Channel);
|
||||
Assert.Equal(AuditKind.InboundAuthFailure, evt.Kind);
|
||||
Assert.Equal(AuditStatus.Failed, evt.Status);
|
||||
Assert.Equal(401, evt.HttpStatus);
|
||||
// Never echo back an unauthenticated principal — middleware suppresses
|
||||
// the framework user resolution on 401/403 paths.
|
||||
Assert.Null(evt.Actor);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task PostToApi_Returning500_Emits_InboundRequest_StatusFailed_HttpStatus500()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var methodName = NewMethodName("server-error");
|
||||
|
||||
using var host = await BuildHostAsync(async ctx =>
|
||||
{
|
||||
// A handler-returned 500 (not a throw) — auth succeeded so Actor
|
||||
// resolution is still allowed; the audit row's Kind stays
|
||||
// InboundRequest (not InboundAuthFailure) and Status flips to
|
||||
// Failed because the response is not a 2xx.
|
||||
ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "integration-svc";
|
||||
ctx.Response.StatusCode = 500;
|
||||
await ctx.Response.WriteAsync("kaboom");
|
||||
});
|
||||
|
||||
var client = host.GetTestClient();
|
||||
var resp = await client.PostAsync(
|
||||
$"/api/{methodName}",
|
||||
new StringContent("{}", Encoding.UTF8, "application/json"));
|
||||
Assert.Equal(System.Net.HttpStatusCode.InternalServerError, resp.StatusCode);
|
||||
|
||||
var evt = await AwaitOneAsync(methodName);
|
||||
Assert.Equal(AuditChannel.ApiInbound, evt.Channel);
|
||||
Assert.Equal(AuditKind.InboundRequest, evt.Kind);
|
||||
Assert.Equal(AuditStatus.Failed, evt.Status);
|
||||
Assert.Equal(500, evt.HttpStatus);
|
||||
Assert.Equal("integration-svc", evt.Actor);
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Timestamp = Google.Protobuf.WellKnownTypes.Timestamp;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Test-side combined-telemetry dispatcher: wraps a production
|
||||
/// <see cref="ICachedCallTelemetryForwarder"/> so the local audit + tracking
|
||||
/// stores still get written, then projects the same packet onto the wire as a
|
||||
/// <see cref="CachedTelemetryBatch"/> and pushes it through the supplied
|
||||
/// <see cref="ISiteStreamAuditClient"/>. The bridge can be composed into the
|
||||
/// existing <see cref="CachedCallLifecycleBridge"/> chain as the
|
||||
/// <see cref="ICachedCallTelemetryForwarder"/> implementation so a single
|
||||
/// observer callback fans out to both halves.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Production wiring keeps the wire push deferred to M6 — the site SQLite hot
|
||||
/// path is the source of truth and a future M6 drain will push the rows
|
||||
/// through the gRPC client. For end-to-end testing today we need a way to
|
||||
/// exercise the central dual-write transaction immediately, so this
|
||||
/// dispatcher synthesises the wire packet inline and round-trips it through
|
||||
/// the stub client. The shape mirrors what the M6 drain will eventually emit.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Best-effort:</b> both the inner forwarder call and the wire push are
|
||||
/// wrapped in independent try/catch blocks. A thrown wire client doesn't
|
||||
/// abort the local writes (the audit row is already in SQLite); a thrown
|
||||
/// local forwarder doesn't abort the wire push (central still gets the
|
||||
/// dual-write attempt).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class CombinedTelemetryDispatcher : ICachedCallTelemetryForwarder
|
||||
{
|
||||
private readonly ICachedCallTelemetryForwarder _inner;
|
||||
private readonly ISiteStreamAuditClient _wireClient;
|
||||
|
||||
public CombinedTelemetryDispatcher(
|
||||
ICachedCallTelemetryForwarder inner,
|
||||
ISiteStreamAuditClient wireClient)
|
||||
{
|
||||
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
|
||||
_wireClient = wireClient ?? throw new ArgumentNullException(nameof(wireClient));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(telemetry);
|
||||
|
||||
// Inner forwarder writes the audit row to SQLite + updates the
|
||||
// tracking store. Best-effort — exceptions are already swallowed
|
||||
// inside the production forwarder, but wrap defensively here too in
|
||||
// case a test substitutes a stricter inner.
|
||||
try
|
||||
{
|
||||
await _inner.ForwardAsync(telemetry, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Swallow — alog.md §7 best-effort contract.
|
||||
}
|
||||
|
||||
// Project the same packet onto the wire and push it through the stub
|
||||
// client. This is the bit a future M6 drain will subsume — until
|
||||
// then the test wraps the two halves into one observer-driven step.
|
||||
try
|
||||
{
|
||||
var batch = new CachedTelemetryBatch();
|
||||
batch.Packets.Add(BuildPacket(telemetry));
|
||||
await _wireClient.IngestCachedTelemetryAsync(batch, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Swallow — the audit row is still in SQLite for a future drain;
|
||||
// the central row will materialise the next time the wire path
|
||||
// is exercised (or via the M6 reconciliation pull).
|
||||
}
|
||||
}
|
||||
|
||||
private static CachedTelemetryPacket BuildPacket(CachedCallTelemetry telemetry)
|
||||
{
|
||||
return new CachedTelemetryPacket
|
||||
{
|
||||
AuditEvent = AuditEventDtoMapper.ToDto(telemetry.Audit),
|
||||
Operational = ToOperationalDto(telemetry.Operational),
|
||||
};
|
||||
}
|
||||
|
||||
private static SiteCallOperationalDto ToOperationalDto(SiteCallOperational op)
|
||||
{
|
||||
var dto = new SiteCallOperationalDto
|
||||
{
|
||||
TrackedOperationId = op.TrackedOperationId.Value.ToString("D"),
|
||||
Channel = op.Channel,
|
||||
Target = op.Target,
|
||||
SourceSite = op.SourceSite,
|
||||
SourceNode = op.SourceNode ?? string.Empty,
|
||||
Status = op.Status,
|
||||
RetryCount = op.RetryCount,
|
||||
LastError = op.LastError ?? string.Empty,
|
||||
CreatedAtUtc = Timestamp.FromDateTime(EnsureUtc(op.CreatedAtUtc)),
|
||||
UpdatedAtUtc = Timestamp.FromDateTime(EnsureUtc(op.UpdatedAtUtc)),
|
||||
};
|
||||
if (op.HttpStatus.HasValue)
|
||||
{
|
||||
dto.HttpStatus = op.HttpStatus.Value;
|
||||
}
|
||||
if (op.TerminalAtUtc.HasValue)
|
||||
{
|
||||
dto.TerminalAtUtc = Timestamp.FromDateTime(EnsureUtc(op.TerminalAtUtc.Value));
|
||||
}
|
||||
return dto;
|
||||
}
|
||||
|
||||
private static DateTime EnsureUtc(DateTime value) =>
|
||||
value.Kind == DateTimeKind.Utc
|
||||
? value
|
||||
: DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc);
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Shared end-to-end harness for the M3 cached-call combined telemetry tests
|
||||
/// (G2/G3/G4). Composes the full pipeline:
|
||||
/// <list type="bullet">
|
||||
/// <item><description>Site-local SQLite <see cref="SqliteAuditWriter"/> (in-memory) +
|
||||
/// <see cref="RingBufferFallback"/> + <see cref="FallbackAuditWriter"/>.</description></item>
|
||||
/// <item><description>Site-local SQLite <see cref="OperationTrackingStore"/> (in-memory).</description></item>
|
||||
/// <item><description>Production <see cref="CachedCallTelemetryForwarder"/> wrapped by a
|
||||
/// test-side <see cref="CombinedTelemetryDispatcher"/> that also pushes each
|
||||
/// packet through the stub gRPC client.</description></item>
|
||||
/// <item><description><see cref="CachedCallLifecycleBridge"/> wired to the
|
||||
/// dispatcher so a single observer call fans out audit + tracking + wire.</description></item>
|
||||
/// <item><description><see cref="DirectActorSiteStreamAuditClient"/> connected
|
||||
/// to an <see cref="AuditLogIngestActor"/> backed by the real
|
||||
/// <see cref="AuditLogRepository"/> + <see cref="SiteCallAuditRepository"/>
|
||||
/// against the per-test <see cref="MsSqlMigrationFixture"/> database.</description></item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Disposal cleans up the in-memory SQLite stores. The Akka actor system is
|
||||
/// owned by the calling <see cref="Akka.TestKit.Xunit2.TestKit"/>; the harness
|
||||
/// only owns the ingest actor IActorRef and the underlying repositories'
|
||||
/// DbContext lifecycle.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class CombinedTelemetryHarness : IAsyncDisposable
|
||||
{
|
||||
public SqliteAuditWriter SqliteWriter { get; }
|
||||
public RingBufferFallback Ring { get; }
|
||||
public FallbackAuditWriter FallbackWriter { get; }
|
||||
public OperationTrackingStore TrackingStore { get; }
|
||||
public CachedCallTelemetryForwarder InnerForwarder { get; }
|
||||
public CombinedTelemetryDispatcher Dispatcher { get; }
|
||||
public CachedCallLifecycleBridge Bridge { get; }
|
||||
public DirectActorSiteStreamAuditClient StubClient { get; }
|
||||
public IActorRef IngestActor { get; }
|
||||
public IServiceProvider ServiceProvider { get; }
|
||||
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
private bool _disposed;
|
||||
|
||||
public CombinedTelemetryHarness(
|
||||
MsSqlMigrationFixture fixture,
|
||||
Akka.TestKit.Xunit2.TestKit testKit,
|
||||
Func<ScadaBridgeDbContext, ISiteCallAuditRepository>? siteCallRepoOverride = null)
|
||||
{
|
||||
_fixture = fixture ?? throw new ArgumentNullException(nameof(fixture));
|
||||
ArgumentNullException.ThrowIfNull(testKit);
|
||||
|
||||
// Site SQLite — unique in-memory database per harness so tests don't share
|
||||
// an audit queue. Mode=Memory + Cache=Shared keeps the file alive for the
|
||||
// lifetime of the writer connection.
|
||||
SqliteWriter = new SqliteAuditWriter(
|
||||
Options.Create(new SqliteAuditWriterOptions
|
||||
{
|
||||
DatabasePath = "ignored",
|
||||
BatchSize = 64,
|
||||
ChannelCapacity = 1024,
|
||||
}),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
connectionStringOverride:
|
||||
$"Data Source=file:cachedcall-g-{Guid.NewGuid():N}?mode=memory&cache=shared");
|
||||
|
||||
Ring = new RingBufferFallback();
|
||||
FallbackWriter = new FallbackAuditWriter(
|
||||
SqliteWriter, Ring, new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
|
||||
TrackingStore = new OperationTrackingStore(
|
||||
Options.Create(new OperationTrackingOptions
|
||||
{
|
||||
// Same shared-in-memory pattern as the audit writer.
|
||||
ConnectionString =
|
||||
$"Data Source=file:tracking-g-{Guid.NewGuid():N}?mode=memory&cache=shared",
|
||||
}),
|
||||
NullLogger<OperationTrackingStore>.Instance);
|
||||
|
||||
// Central wiring: real repositories backed by the MSSQL fixture's DB.
|
||||
ServiceProvider = BuildCentralServiceProvider(siteCallRepoOverride);
|
||||
IngestActor = testKit.Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
|
||||
ServiceProvider,
|
||||
NullLogger<AuditLogIngestActor>.Instance)));
|
||||
|
||||
StubClient = new DirectActorSiteStreamAuditClient(IngestActor);
|
||||
|
||||
// Production forwarder writes the local stores; the dispatcher wraps
|
||||
// it to ALSO push the same packet to central via the stub client.
|
||||
InnerForwarder = new CachedCallTelemetryForwarder(
|
||||
FallbackWriter, TrackingStore, NullLogger<CachedCallTelemetryForwarder>.Instance);
|
||||
Dispatcher = new CombinedTelemetryDispatcher(InnerForwarder, StubClient);
|
||||
|
||||
Bridge = new CachedCallLifecycleBridge(Dispatcher, NullLogger<CachedCallLifecycleBridge>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Convenience: emit the initial submit packet directly through the
|
||||
/// dispatcher (the bridge's hooks fire only for S&F retry-loop
|
||||
/// attempts; submit-row emission happens at the script call site).
|
||||
/// </summary>
|
||||
public Task EmitSubmitAsync(CachedCallTelemetry submit, CancellationToken ct = default) =>
|
||||
Dispatcher.ForwardAsync(submit, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Convenience: route a per-attempt or terminal outcome through the bridge.
|
||||
/// </summary>
|
||||
public Task EmitAttemptAsync(CachedCallAttemptContext context, CancellationToken ct = default) =>
|
||||
Bridge.OnAttemptCompletedAsync(context, ct);
|
||||
|
||||
public ScadaBridgeDbContext CreateReadContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
private IServiceProvider BuildCentralServiceProvider(
|
||||
Func<ScadaBridgeDbContext, ISiteCallAuditRepository>? siteCallRepoOverride)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(opts =>
|
||||
opts.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
|
||||
services.AddScoped<IAuditLogRepository>(sp =>
|
||||
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
if (siteCallRepoOverride is null)
|
||||
{
|
||||
services.AddScoped<ISiteCallAuditRepository>(sp =>
|
||||
new SiteCallAuditRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
}
|
||||
else
|
||||
{
|
||||
services.AddScoped(sp =>
|
||||
siteCallRepoOverride(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
}
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
await SqliteWriter.DisposeAsync().ConfigureAwait(false);
|
||||
await TrackingStore.DisposeAsync().ConfigureAwait(false);
|
||||
if (ServiceProvider is IAsyncDisposable asyncSp)
|
||||
{
|
||||
await asyncSp.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
else if (ServiceProvider is IDisposable sp)
|
||||
{
|
||||
sp.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Shared component-level <see cref="ISiteStreamAuditClient"/> test double that
|
||||
/// short-circuits the gRPC wire and forwards each batch directly to a central
|
||||
/// <see cref="AuditLog.Central.AuditLogIngestActor"/> via Akka <see cref="Futures.Ask"/>.
|
||||
/// Lives under <c>Integration/Infrastructure/</c> so both the M2 sync-call and
|
||||
/// M3 cached-call end-to-end suites can reuse it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The class deliberately mirrors the production <c>SiteStreamGrpcServer</c>
|
||||
/// flow: decode each DTO into the in-process entity, Ask the central ingest
|
||||
/// actor with the matching Akka command, and convert the Akka reply's accepted
|
||||
/// id list into the proto <see cref="IngestAck"/> the telemetry actor / forwarder
|
||||
/// expects. The actor wiring (single-repository vs. <see cref="IServiceProvider"/>
|
||||
/// ctor) lives in the central ingest actor itself — this stub just routes the
|
||||
/// command.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <see cref="FailNextCallCount"/> arms a deterministic number of failures
|
||||
/// before the stub recovers; it applies to BOTH RPCs because the M2 sync-call
|
||||
/// retry behaviour and the M3 cached-telemetry retry behaviour share a single
|
||||
/// SiteAuditTelemetryActor drain. Tests that need to differentiate per-RPC
|
||||
/// failures should reach for a per-test wrapper rather than extending this
|
||||
/// shared infrastructure.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DirectActorSiteStreamAuditClient : ISiteStreamAuditClient
|
||||
{
|
||||
private readonly IActorRef _ingestActor;
|
||||
private int _failsRemaining;
|
||||
private int _callCount;
|
||||
private int _cachedTelemetryCallCount;
|
||||
|
||||
public DirectActorSiteStreamAuditClient(IActorRef ingestActor)
|
||||
{
|
||||
_ingestActor = ingestActor ?? throw new ArgumentNullException(nameof(ingestActor));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When > 0, the next <c>FailNextCallCount</c> invocations of either
|
||||
/// RPC throw to simulate a gRPC error; after that count is exhausted, calls
|
||||
/// succeed normally.
|
||||
/// </summary>
|
||||
public int FailNextCallCount
|
||||
{
|
||||
get => _failsRemaining;
|
||||
set => _failsRemaining = value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Total successful + failed invocations of <see cref="IngestAuditEventsAsync"/>.
|
||||
/// </summary>
|
||||
public int CallCount => Volatile.Read(ref _callCount);
|
||||
|
||||
/// <summary>
|
||||
/// Total successful + failed invocations of <see cref="IngestCachedTelemetryAsync"/>.
|
||||
/// Separate counter so cached-call tests can assert dispatch independently of
|
||||
/// any sync-call traffic going through the same stub.
|
||||
/// </summary>
|
||||
public int CachedTelemetryCallCount => Volatile.Read(ref _cachedTelemetryCallCount);
|
||||
|
||||
public async Task<IngestAck> IngestAuditEventsAsync(AuditEventBatch batch, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _callCount);
|
||||
|
||||
// Atomically consume one of the queued failures, if any. This lets the
|
||||
// test arm a deterministic number of failures before the stub recovers.
|
||||
if (Interlocked.Decrement(ref _failsRemaining) >= 0)
|
||||
{
|
||||
throw new InvalidOperationException("simulated gRPC failure for test");
|
||||
}
|
||||
|
||||
// Clamp at -1 to keep the field bounded under many calls.
|
||||
Interlocked.Exchange(ref _failsRemaining, -1);
|
||||
|
||||
// Decode the proto batch back into AuditEvent records — mirrors what
|
||||
// SiteStreamGrpcServer does before dispatching to the ingest actor.
|
||||
var events = new List<AuditEvent>(batch.Events.Count);
|
||||
foreach (var dto in batch.Events)
|
||||
{
|
||||
events.Add(AuditEventDtoMapper.FromDto(dto));
|
||||
}
|
||||
|
||||
// Ask the central actor; the reply carries the accepted EventIds.
|
||||
var reply = await _ingestActor
|
||||
.Ask<IngestAuditEventsReply>(
|
||||
new IngestAuditEventsCommand(events),
|
||||
TimeSpan.FromSeconds(10))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var ack = new IngestAck();
|
||||
foreach (var id in reply.AcceptedEventIds)
|
||||
{
|
||||
ack.AcceptedEventIds.Add(id.ToString());
|
||||
}
|
||||
return ack;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// M3 dual-write path: decode each <see cref="CachedTelemetryPacket"/> into
|
||||
/// the paired (<see cref="AuditEvent"/>, <see cref="SiteCall"/>) entry and
|
||||
/// Ask the central ingest actor with an <see cref="IngestCachedTelemetryCommand"/>.
|
||||
/// The accepted EventIds returned by the actor's dual-write transaction map
|
||||
/// back into the proto ack.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Uses the shared <see cref="AuditEventDtoMapper.FromDto"/> for the audit half
|
||||
/// and <see cref="SiteCallDtoMapper.FromDto"/> for the SiteCall half — the same
|
||||
/// canonical mappers the production <c>SiteStreamGrpcServer</c> uses.
|
||||
/// </remarks>
|
||||
public async Task<IngestAck> IngestCachedTelemetryAsync(CachedTelemetryBatch batch, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _cachedTelemetryCallCount);
|
||||
|
||||
if (Interlocked.Decrement(ref _failsRemaining) >= 0)
|
||||
{
|
||||
throw new InvalidOperationException("simulated gRPC failure for test");
|
||||
}
|
||||
Interlocked.Exchange(ref _failsRemaining, -1);
|
||||
|
||||
var entries = new List<CachedTelemetryEntry>(batch.Packets.Count);
|
||||
foreach (var packet in batch.Packets)
|
||||
{
|
||||
var audit = AuditEventDtoMapper.FromDto(packet.AuditEvent);
|
||||
var siteCall = SiteCallDtoMapper.FromDto(packet.Operational);
|
||||
entries.Add(new CachedTelemetryEntry(audit, siteCall));
|
||||
}
|
||||
|
||||
var reply = await _ingestActor
|
||||
.Ask<IngestCachedTelemetryReply>(
|
||||
new IngestCachedTelemetryCommand(entries),
|
||||
TimeSpan.FromSeconds(10))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var ack = new IngestAck();
|
||||
foreach (var id in reply.AcceptedEventIds)
|
||||
{
|
||||
ack.AcceptedEventIds.Add(id.ToString());
|
||||
}
|
||||
return ack;
|
||||
}
|
||||
}
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — M4 Bundle E (Task E2): end-to-end audit trail produced by
|
||||
/// the central <see cref="NotificationOutboxActor"/> dispatcher loop. Wires
|
||||
/// the production <see cref="CentralAuditWriter"/> onto the real
|
||||
/// <see cref="AuditLogRepository"/> against the per-class
|
||||
/// <see cref="MsSqlMigrationFixture"/> MSSQL database, drives the dispatcher
|
||||
/// with a stub <see cref="INotificationDeliveryAdapter"/> that yields a
|
||||
/// transient-then-success sequence, and asserts the resulting
|
||||
/// <see cref="AuditChannel.Notification"/>/<see cref="AuditKind.NotifyDeliver"/>
|
||||
/// rows materialise with the expected Attempted/Delivered shape.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The Submit row is normally produced by the site-side <c>Notify.Send</c>
|
||||
/// wrapper (Bundle C); for this E2E we pre-insert a single AuditLog Submit row
|
||||
/// via <see cref="IAuditLogRepository"/> alongside the seeded
|
||||
/// <see cref="Notification"/> row so the assertions can confirm the dispatcher
|
||||
/// emissions slot in alongside it. This keeps the test focused on the
|
||||
/// dispatcher's emission shape without depending on the upstream site path.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Each test uses a unique notification id + source-site id so concurrent
|
||||
/// tests sharing the MSSQL fixture don't interfere. The dispatcher is driven
|
||||
/// deterministically via the internal
|
||||
/// <c>InternalMessages.DispatchTick.Instance</c> sentinel (same pattern the
|
||||
/// existing NotificationOutbox.Tests use).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class NotifyDispatcherAuditTrailTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public NotifyDispatcherAuditTrailTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-e2-notify-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a DI provider that mirrors the production wiring expected by
|
||||
/// <see cref="NotificationOutboxActor"/>: scoped EF-backed
|
||||
/// <see cref="INotificationOutboxRepository"/> + <see cref="INotificationRepository"/>
|
||||
/// + the supplied <see cref="INotificationDeliveryAdapter"/>. The
|
||||
/// <see cref="IAuditLogRepository"/> registration powers the
|
||||
/// <see cref="CentralAuditWriter"/> the actor will emit through.
|
||||
/// </summary>
|
||||
private IServiceProvider BuildServiceProvider(INotificationDeliveryAdapter adapter)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(opts =>
|
||||
opts.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
|
||||
services.AddScoped<INotificationOutboxRepository>(sp =>
|
||||
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddScoped<INotificationRepository>(sp =>
|
||||
new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddScoped<IAuditLogRepository>(sp =>
|
||||
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stub adapter that yields the next outcome from a configurable queue per
|
||||
/// call. Lets a single dispatch sweep exercise the transient-then-success
|
||||
/// transition by alternating <see cref="DeliveryResult.TransientFailure"/>
|
||||
/// and <see cref="DeliveryResult.Success"/>.
|
||||
/// </summary>
|
||||
private sealed class QueuedOutcomeAdapter : INotificationDeliveryAdapter
|
||||
{
|
||||
private readonly Queue<DeliveryOutcome> _outcomes;
|
||||
public int CallCount;
|
||||
|
||||
public QueuedOutcomeAdapter(params DeliveryOutcome[] outcomes)
|
||||
{
|
||||
_outcomes = new Queue<DeliveryOutcome>(outcomes);
|
||||
}
|
||||
|
||||
public NotificationType Type => NotificationType.Email;
|
||||
|
||||
public Task<DeliveryOutcome> DeliverAsync(
|
||||
Notification notification, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Interlocked.Increment(ref CallCount);
|
||||
// Defensive — if a test under-supplies outcomes we surface the
|
||||
// problem as an explicit transient failure rather than throwing
|
||||
// (the dispatcher would log + skip the notification but the audit
|
||||
// assertions would be misleading).
|
||||
var outcome = _outcomes.Count > 0
|
||||
? _outcomes.Dequeue()
|
||||
: DeliveryOutcome.Transient("test stub out of outcomes");
|
||||
return Task.FromResult(outcome);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a single SMTP configuration row so the dispatcher's
|
||||
/// <c>ResolveRetryPolicyAsync</c> sees a real (maxRetries, retryDelay)
|
||||
/// pair rather than the conservative fallback. A tiny positive RetryDelay
|
||||
/// means a transient outcome's <c>NextAttemptAt</c> is immediately due —
|
||||
/// useful so the SECOND DispatchTick re-claims the row without waiting.
|
||||
/// NO-002: the dispatcher now clamps a non-positive RetryDelay to the
|
||||
/// 1-minute fallback to avoid burn-looping on transient failures, so this
|
||||
/// must be a strictly positive value (1 ms is fine for tests).
|
||||
/// </summary>
|
||||
private async Task SeedSmtpConfigAsync(int maxRetries = 5)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
ctx.SmtpConfigurations.Add(new SmtpConfiguration(
|
||||
"smtp.example.com", "Basic", "noreply@example.com")
|
||||
{
|
||||
MaxRetries = maxRetries,
|
||||
RetryDelay = TimeSpan.FromMilliseconds(1),
|
||||
});
|
||||
await ctx.SaveChangesAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds the Pending outbox row the dispatcher will claim. Using a fixed
|
||||
/// caller-supplied <c>notificationId</c> so the test can later query the
|
||||
/// AuditLog by <see cref="AuditEvent.CorrelationId"/> = notificationId.
|
||||
/// </summary>
|
||||
private async Task<Notification> SeedNotificationAsync(
|
||||
Guid notificationId, string siteId, string listName = "ops-team")
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var n = new Notification(
|
||||
notificationId.ToString("D"),
|
||||
NotificationType.Email,
|
||||
listName,
|
||||
"Tank overflow",
|
||||
"Tank 3 level critical",
|
||||
siteId)
|
||||
{
|
||||
SourceInstanceId = "Plant.Pump42",
|
||||
SourceScript = "AlarmScript",
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1),
|
||||
};
|
||||
ctx.Notifications.Add(n);
|
||||
await ctx.SaveChangesAsync();
|
||||
return n;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pre-inserts the Submit AuditLog row that the site-side Notify.Send
|
||||
/// wrapper would have emitted (Bundle C). Keeps the assertions on the
|
||||
/// dispatcher emissions intact without depending on the upstream site
|
||||
/// path.
|
||||
/// </summary>
|
||||
private async Task SeedSubmitAuditRowAsync(Guid notificationId, string siteId)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
var submitEvt = new AuditEvent
|
||||
{
|
||||
EventId = Guid.NewGuid(),
|
||||
OccurredAtUtc = DateTime.UtcNow.AddMinutes(-1),
|
||||
Channel = AuditChannel.Notification,
|
||||
Kind = AuditKind.NotifySend,
|
||||
CorrelationId = notificationId,
|
||||
SourceSiteId = siteId,
|
||||
SourceInstanceId = "Plant.Pump42",
|
||||
SourceScript = "AlarmScript",
|
||||
Target = "ops-team",
|
||||
Status = AuditStatus.Submitted,
|
||||
ForwardState = AuditForwardState.Forwarded,
|
||||
IngestedAtUtc = DateTime.UtcNow.AddMinutes(-1),
|
||||
};
|
||||
await repo.InsertIfNotExistsAsync(submitEvt);
|
||||
}
|
||||
|
||||
private static NotificationOutboxOptions LongDispatchOptions() =>
|
||||
// 1h dispatch + 24h purge so PreStart's timers never fire during the
|
||||
// test; the test drives the dispatcher with explicit DispatchTick.
|
||||
new()
|
||||
{
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromDays(1),
|
||||
};
|
||||
|
||||
[SkippableFact]
|
||||
public async Task NotifyDispatcher_FailThenSuccess_Emits_TwoAttempts_OneDelivered_Terminal()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var notificationId = Guid.NewGuid();
|
||||
await SeedSmtpConfigAsync(maxRetries: 5);
|
||||
await SeedNotificationAsync(notificationId, siteId);
|
||||
await SeedSubmitAuditRowAsync(notificationId, siteId);
|
||||
|
||||
var adapter = new QueuedOutcomeAdapter(
|
||||
DeliveryOutcome.Transient("smtp 421 try again"),
|
||||
DeliveryOutcome.Success("ops@example.com"));
|
||||
var serviceProvider = BuildServiceProvider(adapter);
|
||||
var auditWriter = new CentralAuditWriter(
|
||||
serviceProvider,
|
||||
NullLogger<CentralAuditWriter>.Instance);
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
||||
serviceProvider,
|
||||
LongDispatchOptions(),
|
||||
(ICentralAuditWriter)auditWriter,
|
||||
NullLogger<NotificationOutboxActor>.Instance)));
|
||||
|
||||
// First tick: transient failure → one Attempted row, no terminal row.
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
var rows = await repo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 50));
|
||||
// 1 Submit + 1 Attempted = 2 rows so far.
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.Single(rows, r => r.Kind == AuditKind.NotifyDeliver
|
||||
&& r.Status == AuditStatus.Attempted);
|
||||
Assert.Single(rows, r => r.Kind == AuditKind.NotifySend);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
|
||||
// Second tick: success → second Attempted + one Delivered terminal.
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
var rows = await repo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 50));
|
||||
// 1 Submit + 2 Attempted + 1 Delivered terminal = 4 rows.
|
||||
Assert.InRange(rows.Count, 3, 4);
|
||||
var notifyDeliverRows = rows
|
||||
.Where(r => r.Kind == AuditKind.NotifyDeliver)
|
||||
.ToList();
|
||||
Assert.Equal(2, notifyDeliverRows.Count(r => r.Status == AuditStatus.Attempted));
|
||||
var terminal = Assert.Single(notifyDeliverRows, r => r.Status == AuditStatus.Delivered);
|
||||
// All NotifyDeliver rows correlate to the original notification id.
|
||||
Assert.All(notifyDeliverRows, r => Assert.Equal(notificationId, r.CorrelationId));
|
||||
Assert.Equal("ops-team", terminal.Target);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
|
||||
// Operational Notifications table mirrors the audit outcome.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var n = await ctx.Notifications.SingleAsync(
|
||||
row => row.NotificationId == notificationId.ToString("D"));
|
||||
Assert.Equal(NotificationStatus.Delivered, n.Status);
|
||||
Assert.NotNull(n.DeliveredAt);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task NotifyDispatcher_AuditWriter_Throws_DeliveryStillSucceeds()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
var notificationId = Guid.NewGuid();
|
||||
await SeedSmtpConfigAsync(maxRetries: 5);
|
||||
await SeedNotificationAsync(notificationId, siteId);
|
||||
|
||||
var adapter = new QueuedOutcomeAdapter(
|
||||
DeliveryOutcome.Success("ops@example.com"));
|
||||
var serviceProvider = BuildServiceProvider(adapter);
|
||||
|
||||
// ALWAYS-throw writer wired in place of the production
|
||||
// CentralAuditWriter. The dispatcher MUST still deliver the
|
||||
// notification and persist the terminal Delivered transition
|
||||
// regardless of the audit subsystem being down (alog.md §13).
|
||||
var throwingWriter = new ThrowingCentralAuditWriter();
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
||||
serviceProvider,
|
||||
LongDispatchOptions(),
|
||||
(ICentralAuditWriter)throwingWriter,
|
||||
NullLogger<NotificationOutboxActor>.Instance)));
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
// The Notifications table is the operational source of truth — assert
|
||||
// it transitions to Delivered even though every audit write threw.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var n = await ctx.Notifications.SingleAsync(
|
||||
row => row.NotificationId == notificationId.ToString("D"));
|
||||
Assert.Equal(NotificationStatus.Delivered, n.Status);
|
||||
Assert.NotNull(n.DeliveredAt);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
|
||||
// The writer was attempted (at least once for the Attempted row, plus
|
||||
// once for the Delivered terminal) — proves the dispatcher tried to
|
||||
// emit and absorbed the throws rather than aborting the action.
|
||||
Assert.True(throwingWriter.AttemptCount >= 2,
|
||||
$"Expected the dispatcher to attempt audit writes; saw {throwingWriter.AttemptCount}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test-only <see cref="ICentralAuditWriter"/> that ALWAYS throws on
|
||||
/// <see cref="WriteAsync"/>. Used to verify the dispatcher's defensive
|
||||
/// try/catch contract (alog.md §13) — audit failures must NEVER abort
|
||||
/// the user-facing notification delivery.
|
||||
/// </summary>
|
||||
private sealed class ThrowingCentralAuditWriter : ICentralAuditWriter
|
||||
{
|
||||
private int _attemptCount;
|
||||
public int AttemptCount => Volatile.Read(ref _attemptCount);
|
||||
|
||||
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _attemptCount);
|
||||
throw new InvalidOperationException(
|
||||
"test-only ThrowingCentralAuditWriter — audit subsystem unavailable");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle F (#23 M6-T10) end-to-end test for the central-outage + reconciliation
|
||||
/// recovery loop. Wires the real site SQLite hot-path
|
||||
/// (<see cref="SqliteAuditWriter"/>) and the central <see cref="SiteAuditReconciliationActor"/>
|
||||
/// with an <see cref="AuditLogIngestActor"/> backed by the real
|
||||
/// <see cref="AuditLogRepository"/> on the per-test <see cref="MsSqlMigrationFixture"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The push path is deliberately omitted here: the brief models a sustained
|
||||
/// central outage where the site queue grows unbounded in Pending, then a
|
||||
/// reconciliation pull eventually drains everything once central comes back.
|
||||
/// We reuse the production <see cref="IPullAuditEventsClient"/> seam (Bundle B)
|
||||
/// with a test-only stub that wraps the same <see cref="ISiteAuditQueue.ReadPendingSinceAsync"/>
|
||||
/// surface a real central-side gRPC client would hit, so the test is exercising
|
||||
/// the actor's pull/ingest/mark-reconciled state machine end-to-end against
|
||||
/// the real repository.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <see cref="CombinedTelemetryHarness"/> from M3 is push-only — it has no
|
||||
/// reconciliation puller — so we build the smaller stub inline rather than
|
||||
/// retrofitting the shared harness with a code path it doesn't otherwise
|
||||
/// need.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public OutageReconciliationTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test-only <see cref="IPullAuditEventsClient"/> that mirrors how the
|
||||
/// production central-side gRPC client will hit the site: read a batch
|
||||
/// from <see cref="ISiteAuditQueue.ReadPendingSinceAsync"/>, then commit
|
||||
/// via <see cref="ISiteAuditQueue.MarkReconciledAsync"/> once the central
|
||||
/// repository accepts the rows. The Ask-based central path is wired by
|
||||
/// the caller — we just expose the queue surface.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The production wire shape will be:
|
||||
/// central PullAuditEvents RPC → site SiteStreamGrpcServer.PullAuditEvents
|
||||
/// → ISiteAuditQueue.ReadPendingSinceAsync → marshal proto → reply
|
||||
/// followed by central InsertIfNotExistsAsync per row, then the site flips
|
||||
/// the row to Reconciled on the next pull cycle. The stub collapses the
|
||||
/// two halves (pull + commit) because the actor under test (the
|
||||
/// reconciliation actor) is the side that drives both via the
|
||||
/// IPullAuditEventsClient seam — committing back to the site after the
|
||||
/// repository write is the reconciliation-actor invariant we want to
|
||||
/// observe end-to-end.
|
||||
/// </remarks>
|
||||
private sealed class QueueBackedPullClient : IPullAuditEventsClient
|
||||
{
|
||||
private readonly ISiteAuditQueue _siteQueue;
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public QueueBackedPullClient(ISiteAuditQueue siteQueue)
|
||||
{
|
||||
_siteQueue = siteQueue ?? throw new ArgumentNullException(nameof(siteQueue));
|
||||
}
|
||||
|
||||
public async Task<PullAuditEventsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
||||
{
|
||||
CallCount++;
|
||||
|
||||
var rows = await _siteQueue
|
||||
.ReadPendingSinceAsync(sinceUtc, batchSize, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Commit immediately on the site side — once the actor has the
|
||||
// batch in hand it will InsertIfNotExistsAsync centrally; if the
|
||||
// central insert later throws on a specific row, idempotency
|
||||
// guarantees the next pull cycle does NOT re-fetch the row (it's
|
||||
// already Reconciled on the site) but also does not surface the
|
||||
// failure here. The brief calls this "ack-after-persist" — the
|
||||
// production gRPC server will flip to Reconciled inside its
|
||||
// PullAuditEvents handler after the central side has acknowledged
|
||||
// (per Bundle A's race-fix, central is idempotent on EventId).
|
||||
//
|
||||
// MoreAvailable is true iff the read filled the batch — the actor
|
||||
// uses this to decide whether to follow up on the next tick.
|
||||
if (rows.Count > 0)
|
||||
{
|
||||
var ids = rows.Select(e => e.EventId).ToList();
|
||||
await _siteQueue.MarkReconciledAsync(ids, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return new PullAuditEventsResponse(rows, MoreAvailable: rows.Count >= batchSize);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory enumerator returning a fixed single-site list — mirrors the
|
||||
/// pattern used in <c>SiteAuditReconciliationActorTests</c>.
|
||||
/// </summary>
|
||||
private sealed class StaticEnumerator : ISiteEnumerator
|
||||
{
|
||||
private readonly IReadOnlyList<SiteEntry> _sites;
|
||||
public StaticEnumerator(params SiteEntry[] sites) => _sites = sites;
|
||||
public Task<IReadOnlyList<SiteEntry>> EnumerateAsync(CancellationToken ct = default) =>
|
||||
Task.FromResult(_sites);
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext CreateContext() =>
|
||||
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString).Options);
|
||||
|
||||
private static AuditEvent NewEvent(string siteId, DateTime occurredAt) => new()
|
||||
{
|
||||
EventId = Guid.NewGuid(),
|
||||
OccurredAtUtc = occurredAt,
|
||||
Channel = AuditChannel.ApiOutbound,
|
||||
Kind = AuditKind.ApiCall,
|
||||
Status = AuditStatus.Delivered,
|
||||
SourceSiteId = siteId,
|
||||
Target = "external-system-a/method",
|
||||
};
|
||||
|
||||
private SqliteAuditWriter CreateInMemorySqliteWriter() =>
|
||||
new SqliteAuditWriter(
|
||||
Options.Create(new SqliteAuditWriterOptions
|
||||
{
|
||||
DatabasePath = "ignored",
|
||||
BatchSize = 64,
|
||||
ChannelCapacity = 4096,
|
||||
}),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
connectionStringOverride:
|
||||
$"Data Source=file:outage-{Guid.NewGuid():N}?mode=memory&cache=shared");
|
||||
|
||||
private (IServiceProvider Sp, IActorRef Ingest) BuildCentralPipeline()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(opts =>
|
||||
opts.UseSqlServer(_fixture.ConnectionString));
|
||||
services.AddScoped<IAuditLogRepository>(sp =>
|
||||
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var ingest = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
|
||||
sp,
|
||||
NullLogger<AuditLogIngestActor>.Instance)));
|
||||
return (sp, ingest);
|
||||
}
|
||||
|
||||
private static SiteAuditReconciliationOptions FastTickOptions(int batchSize = 256) => new()
|
||||
{
|
||||
ReconciliationIntervalSeconds = 300,
|
||||
ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||
BatchSize = batchSize,
|
||||
StalledAfterNonDrainingCycles = 2,
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 1. CentralOutage_200Events_Buffer_Then_Reconciliation_Catches_Up_NoDuplicates
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task CentralOutage_200Events_Buffer_Then_Reconciliation_Catches_Up_NoDuplicates()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = "outage-recon-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
// Step 1: site accumulates 200 audit events during the simulated
|
||||
// central outage. The push path is NOT wired here — every row stays
|
||||
// Pending in the site SQLite store until reconciliation runs.
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var baseOccurred = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
|
||||
const int totalEvents = 200;
|
||||
var written = new List<AuditEvent>(totalEvents);
|
||||
|
||||
for (int i = 0; i < totalEvents; i++)
|
||||
{
|
||||
// Strictly monotonic OccurredAtUtc so the cursor can advance
|
||||
// deterministically batch-by-batch — mirrors how a real script
|
||||
// workload generates timestamps in wall-clock order.
|
||||
var evt = NewEvent(siteId, baseOccurred.AddMilliseconds(i));
|
||||
written.Add(evt);
|
||||
await sqliteWriter.WriteAsync(evt);
|
||||
}
|
||||
|
||||
// Sanity: every row is Pending (no push path wired, so nothing has
|
||||
// been Forwarded or Reconciled yet).
|
||||
var pending = await sqliteWriter.ReadPendingAsync(totalEvents + 10);
|
||||
Assert.Equal(totalEvents, pending.Count);
|
||||
|
||||
// Step 2: central comes online — wire the ingest actor + reconciliation
|
||||
// actor. The pull client wraps the site queue directly (the production
|
||||
// shape is one RPC call); each pull advances the actor's cursor and
|
||||
// flips rows on the site to Reconciled.
|
||||
var (sp, ingest) = BuildCentralPipeline();
|
||||
await using (sp as IAsyncDisposable ?? throw new InvalidOperationException())
|
||||
{
|
||||
var pullClient = new QueueBackedPullClient(sqliteWriter);
|
||||
var enumerator = new StaticEnumerator(new SiteEntry(siteId, "http://test:8083"));
|
||||
|
||||
// BatchSize = 64 so the actor needs ~4 ticks to drain 200 rows.
|
||||
// The "after 5 minutes" wording in the brief is satisfied by the
|
||||
// fast-tick override (100 ms per tick) plus AwaitAssert giving
|
||||
// the actor up to ~30 seconds to settle in real time.
|
||||
var opts = FastTickOptions(batchSize: 64);
|
||||
|
||||
// Standalone DI scope for the reconciliation actor (it shares the
|
||||
// ingest actor's IServiceProvider so both writers see the same
|
||||
// EF context configuration).
|
||||
var reconciliationActor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
|
||||
enumerator,
|
||||
pullClient,
|
||||
sp,
|
||||
Options.Create(opts),
|
||||
NullLogger<SiteAuditReconciliationActor>.Instance)));
|
||||
|
||||
// Step 3: assert central AuditLog has all 200 rows after the
|
||||
// actor drains. Polling the real MSSQL repository — the test
|
||||
// fixture has its own database so a count restricted to this
|
||||
// SourceSiteId is exact.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var count = await ctx.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.CountAsync();
|
||||
Assert.Equal(totalEvents, count);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(30),
|
||||
interval: TimeSpan.FromMilliseconds(200));
|
||||
|
||||
// Step 4: assert site rows flipped to Reconciled.
|
||||
// ReadPendingAsync only returns Pending rows; after a full drain
|
||||
// it must be empty.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var stillPending = await sqliteWriter.ReadPendingAsync(totalEvents + 10);
|
||||
Assert.Empty(stillPending);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(10),
|
||||
interval: TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Step 5: assert no duplicates by EventId — central must have
|
||||
// exactly the 200 rows we wrote at the site (one row per EventId).
|
||||
await using var verify = CreateContext();
|
||||
var centralIds = await verify.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.Select(e => e.EventId)
|
||||
.ToListAsync();
|
||||
Assert.Equal(totalEvents, centralIds.Count);
|
||||
Assert.Equal(totalEvents, centralIds.Distinct().Count());
|
||||
// And every EventId we wrote at the site is present centrally.
|
||||
Assert.True(written.All(w => centralIds.Contains(w.EventId)),
|
||||
"every site-written EventId should be present centrally.");
|
||||
|
||||
// Tear the actor down before disposing the harness; the actor's
|
||||
// PostStop cancels its scheduled timer.
|
||||
Sys.Stop(reconciliationActor);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 2. ReconciliationPull_Idempotent_Across_Two_Cycles
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ReconciliationPull_Idempotent_Across_Two_Cycles()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = "outage-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
const int totalEvents = 50;
|
||||
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var baseOccurred = new DateTime(2026, 5, 20, 13, 0, 0, DateTimeKind.Utc);
|
||||
for (int i = 0; i < totalEvents; i++)
|
||||
{
|
||||
await sqliteWriter.WriteAsync(NewEvent(siteId, baseOccurred.AddMilliseconds(i)));
|
||||
}
|
||||
|
||||
var (sp, _) = BuildCentralPipeline();
|
||||
await using (sp as IAsyncDisposable ?? throw new InvalidOperationException())
|
||||
{
|
||||
var pullClient = new QueueBackedPullClient(sqliteWriter);
|
||||
var enumerator = new StaticEnumerator(new SiteEntry(siteId, "http://test:8083"));
|
||||
|
||||
var reconciliationActor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
|
||||
enumerator,
|
||||
pullClient,
|
||||
sp,
|
||||
Options.Create(FastTickOptions()),
|
||||
NullLogger<SiteAuditReconciliationActor>.Instance)));
|
||||
|
||||
// Wait for the first drain cycle to complete.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var count = await ctx.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.CountAsync();
|
||||
Assert.Equal(totalEvents, count);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(30),
|
||||
interval: TimeSpan.FromMilliseconds(200));
|
||||
|
||||
// Wait for additional pull cycles to fire — the actor ticks every
|
||||
// 100 ms so a 1 s settle leaves the actor with at least ~5 ticks
|
||||
// past the initial drain. Each subsequent tick must be a no-op
|
||||
// because every row is now Reconciled and outside the
|
||||
// ReadPendingSinceAsync filter.
|
||||
var callsAfterDrain = pullClient.CallCount;
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(800));
|
||||
Assert.True(pullClient.CallCount > callsAfterDrain,
|
||||
$"expected additional pull calls after drain to validate idempotency, got {pullClient.CallCount} after {callsAfterDrain}");
|
||||
|
||||
// Central count must still be exactly totalEvents — no duplicates
|
||||
// even though the cursor + read-Reconciled-too semantics could
|
||||
// theoretically re-fetch on the second cycle.
|
||||
await using var verify = CreateContext();
|
||||
var rows = await verify.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
Assert.Equal(totalEvents, rows.Count);
|
||||
Assert.Equal(totalEvents, rows.Select(r => r.EventId).Distinct().Count());
|
||||
|
||||
Sys.Stop(reconciliationActor);
|
||||
}
|
||||
}
|
||||
}
|
||||
+626
@@ -0,0 +1,626 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using ZB.MOM.WW.ScadaBridge.InboundAPI;
|
||||
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log #23 — <b>ParentExecutionId cross-execution correlation</b> headline
|
||||
/// end-to-end suite. Verifies the inbound-API → routed-site-script bridge: an
|
||||
/// inbound HTTP request runs an inbound method script that calls
|
||||
/// <c>Route.Call</c> into a site instance; the routed site script does a sync
|
||||
/// <c>ExternalSystem.Call</c>, a cached call and a <c>Notify.Send</c>. Every
|
||||
/// audit row the routed run produces — site + central, sync + cached lifecycle
|
||||
/// + <c>NotifySend</c>/<c>NotifyDeliver</c> — must carry
|
||||
/// <see cref="AuditEvent.ParentExecutionId"/> equal to the inbound request's
|
||||
/// <see cref="AuditEvent.ExecutionId"/>, while the routed run has its own
|
||||
/// distinct <see cref="AuditEvent.ExecutionId"/> and the inbound
|
||||
/// <see cref="AuditKind.InboundRequest"/> row is top-level
|
||||
/// (<c>ParentExecutionId = NULL</c>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is the integration-level counterpart to <see cref="ExecutionIdCorrelationTests"/>:
|
||||
/// where that suite drives a single <see cref="ScriptRuntimeContext"/> run and
|
||||
/// asserts the shared per-run <c>ExecutionId</c>, this suite spans <b>two</b>
|
||||
/// executions on opposite sides of the inbound→routed bridge and asserts the
|
||||
/// cross-execution <c>ParentExecutionId</c> linkage plus
|
||||
/// <see cref="IAuditLogRepository.GetExecutionTreeAsync"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The bridge is exercised through the genuine production glue:
|
||||
/// <list type="bullet">
|
||||
/// <item><description>the real <see cref="AuditWriteMiddleware"/> in a
|
||||
/// Microsoft.AspNetCore.TestHost pipeline — mints the inbound request's
|
||||
/// per-request <c>ExecutionId</c> once, stashes it on
|
||||
/// <see cref="HttpContext.Items"/>, and emits the top-level
|
||||
/// <see cref="AuditKind.InboundRequest"/> row via the real
|
||||
/// <see cref="CentralAuditWriter"/>;</description></item>
|
||||
/// <item><description>the real <see cref="InboundScriptExecutor"/> +
|
||||
/// <see cref="RouteHelper"/> — the executor binds the stashed inbound
|
||||
/// <c>ExecutionId</c> via <see cref="RouteHelper.WithParentExecutionId"/>, so a
|
||||
/// <c>Route.To(...).Call(...)</c> inside the inbound script builds a
|
||||
/// <see cref="RouteToCallRequest"/> carrying
|
||||
/// <see cref="RouteToCallRequest.ParentExecutionId"/>.</description></item>
|
||||
/// </list>
|
||||
/// Only the cross-cluster routing transport is substituted: the test
|
||||
/// <see cref="BridgingInstanceRouter"/> stands in for
|
||||
/// <c>CommunicationServiceInstanceRouter</c> exactly as the production site
|
||||
/// (<c>DeploymentManagerActor</c> → <c>ScriptActor</c> → <c>ScriptExecutionActor</c>)
|
||||
/// would — it reads <see cref="RouteToCallRequest.ParentExecutionId"/> off the
|
||||
/// wire request and threads it into the routed <see cref="ScriptRuntimeContext"/>
|
||||
/// as <c>parentExecutionId</c>. A multi-node cluster is out of scope for an
|
||||
/// in-process test (mirroring <c>SiteAuditPushFlowTests</c>'s relay).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The central audit store is the real <see cref="AuditLogRepository"/> over the
|
||||
/// per-class <see cref="MsSqlMigrationFixture"/> MSSQL database; the routed run's
|
||||
/// site rows reach it through the real <see cref="SqliteAuditWriter"/> hot-path +
|
||||
/// <see cref="SiteAuditTelemetryActor"/> drain, the cached lifecycle rows through
|
||||
/// the production <see cref="CachedCallTelemetryForwarder"/>, and the
|
||||
/// <c>NotifyDeliver</c> rows through the real central
|
||||
/// <see cref="NotificationOutboxActor"/> dispatcher.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class ParentExecutionIdCorrelationTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public ParentExecutionIdCorrelationTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private const string RoutedInstanceCode = "Plant.Pump42";
|
||||
private const string RoutedScriptName = "OnInboundRouted";
|
||||
private const string ExternalSystemName = "ERP";
|
||||
private const string ExternalMethodName = "GetOrder";
|
||||
private const string NotifyListName = "ops-team";
|
||||
|
||||
/// <summary>Per-run site id (Guid suffix) so concurrent tests sharing the MSSQL fixture stay isolated.</summary>
|
||||
private static string NewSiteId() =>
|
||||
"test-parentexec-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task InboundRoutedRun_AllRoutedRows_CarryInboundExecutionId_AsParentExecutionId()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
|
||||
// ── Central — repository + ingest actor + audit writer over the MSSQL fixture ──
|
||||
var centralServices = new ServiceCollection();
|
||||
centralServices.AddDbContext<ScadaBridgeDbContext>(opts =>
|
||||
opts.UseSqlServer(_fixture.ConnectionString)
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
|
||||
centralServices.AddScoped<IAuditLogRepository>(sp =>
|
||||
new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
centralServices.AddScoped<ISiteCallAuditRepository>(sp =>
|
||||
new SiteCallAuditRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
centralServices.AddScoped<INotificationOutboxRepository>(sp =>
|
||||
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
centralServices.AddScoped<INotificationRepository>(sp =>
|
||||
new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
// The NotifyDeliver dispatch path runs through this same long-lived
|
||||
// provider — a stub adapter that always reports a successful delivery.
|
||||
centralServices.AddScoped<INotificationDeliveryAdapter>(_ => new AlwaysDeliversAdapter());
|
||||
await using var centralProvider = centralServices.BuildServiceProvider();
|
||||
|
||||
var ingestActor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
|
||||
(IServiceProvider)centralProvider,
|
||||
NullLogger<AuditLogIngestActor>.Instance)));
|
||||
var centralAuditWriter = new CentralAuditWriter(
|
||||
centralProvider, NullLogger<CentralAuditWriter>.Instance);
|
||||
|
||||
// ── Site — SQLite audit writer (hot-path) drained to central by the
|
||||
// real SiteAuditTelemetryActor through the stub gRPC client. The sync
|
||||
// ApiCall row and the NotifySend row flow through this chain. ──
|
||||
await using var sqliteWriter = new SqliteAuditWriter(
|
||||
Options.Create(new SqliteAuditWriterOptions
|
||||
{
|
||||
DatabasePath = "ignored",
|
||||
BatchSize = 64,
|
||||
ChannelCapacity = 1024,
|
||||
}),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
connectionStringOverride:
|
||||
$"Data Source=file:auditlog-parentexec-{Guid.NewGuid():N}?mode=memory&cache=shared");
|
||||
var ring = new RingBufferFallback();
|
||||
var siteAuditWriter = new FallbackAuditWriter(
|
||||
sqliteWriter, ring, new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
|
||||
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
|
||||
(ISiteAuditQueue)sqliteWriter,
|
||||
stubClient,
|
||||
Options.Create(new SiteAuditTelemetryOptions
|
||||
{
|
||||
BatchSize = 256,
|
||||
BusyIntervalSeconds = 1,
|
||||
IdleIntervalSeconds = 1,
|
||||
}),
|
||||
NullLogger<SiteAuditTelemetryActor>.Instance)));
|
||||
|
||||
// Cached-call telemetry: production forwarder + dispatcher that also
|
||||
// pushes each combined packet through the stub client into the central
|
||||
// dual-write transaction (same wiring CombinedTelemetryHarness uses).
|
||||
var cachedForwarder = new CombinedTelemetryDispatcher(
|
||||
new CachedCallTelemetryForwarder(
|
||||
siteAuditWriter, trackingStore: null,
|
||||
NullLogger<CachedCallTelemetryForwarder>.Instance),
|
||||
stubClient);
|
||||
|
||||
// Site Store-and-Forward — Notify.Send buffers a NotificationSubmit here.
|
||||
using var safKeepAlive = new Microsoft.Data.Sqlite.SqliteConnection(
|
||||
$"Data Source=parentexec-saf-{Guid.NewGuid():N};Mode=Memory;Cache=Shared");
|
||||
safKeepAlive.Open();
|
||||
var safStorage = new StoreAndForwardStorage(
|
||||
safKeepAlive.ConnectionString, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await safStorage.InitializeAsync();
|
||||
var storeAndForward = new StoreAndForwardService(
|
||||
safStorage,
|
||||
new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 3,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
},
|
||||
NullLogger<StoreAndForwardService>.Instance);
|
||||
|
||||
// ── Outbound external-system client (routed run): sync Call succeeds,
|
||||
// CachedCall completes immediately (WasBuffered=false) so the script
|
||||
// helper emits the Submit + Attempted + CachedResolve lifecycle. ──
|
||||
var externalClient = Substitute.For<IExternalSystemClient>();
|
||||
externalClient
|
||||
.CallAsync(ExternalSystemName, ExternalMethodName,
|
||||
Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ExternalCallResult(true, "{\"ok\":true}", null));
|
||||
externalClient
|
||||
.CachedCallAsync(ExternalSystemName, ExternalMethodName,
|
||||
Arg.Any<IReadOnlyDictionary<string, object?>?>(),
|
||||
Arg.Any<string?>(), Arg.Any<CancellationToken>(),
|
||||
Arg.Any<ZB.MOM.WW.ScadaBridge.Commons.Types.TrackedOperationId?>(),
|
||||
Arg.Any<Guid?>(), Arg.Any<string?>(), Arg.Any<Guid?>())
|
||||
.Returns(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
|
||||
|
||||
// ── The routing transport stand-in: builds the routed ScriptRuntimeContext
|
||||
// carrying RouteToCallRequest.ParentExecutionId — exactly what the
|
||||
// production site handler (DeploymentManagerActor) does. ──
|
||||
var router = new BridgingInstanceRouter(
|
||||
siteId,
|
||||
externalClient,
|
||||
siteAuditWriter,
|
||||
cachedForwarder,
|
||||
storeAndForward);
|
||||
|
||||
// ── The inbound API method script: it calls Route.Call into the site
|
||||
// instance. The real InboundScriptExecutor binds the inbound request's
|
||||
// ExecutionId onto the RouteHelper, so the routed call carries it as
|
||||
// ParentExecutionId. ──
|
||||
var inboundMethod = new ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi.ApiMethod(
|
||||
"submitOrder",
|
||||
$"return await Route.To(\"{RoutedInstanceCode}\").Call(\"{RoutedScriptName}\", new {{ order = 7 }});");
|
||||
var locator = Substitute.For<IInstanceLocator>();
|
||||
locator.GetSiteIdForInstanceAsync(RoutedInstanceCode, Arg.Any<CancellationToken>())
|
||||
.Returns(siteId);
|
||||
var scriptExecutor = new InboundScriptExecutor(
|
||||
NullLogger<InboundScriptExecutor>.Instance,
|
||||
new ServiceCollection().BuildServiceProvider());
|
||||
Assert.True(scriptExecutor.CompileAndRegister(inboundMethod));
|
||||
|
||||
// ── Act — issue the inbound HTTP request through a TestHost pipeline
|
||||
// fronted by the real AuditWriteMiddleware. The endpoint handler reads
|
||||
// the middleware-stashed inbound ExecutionId and runs the inbound
|
||||
// method script with it as parentExecutionId. ──
|
||||
using var host = await BuildInboundHostAsync(centralAuditWriter, async ctx =>
|
||||
{
|
||||
var inboundExecutionId = (Guid)ctx.Items[AuditWriteMiddleware.InboundExecutionIdItemKey]!;
|
||||
var route = new RouteHelper(locator, router);
|
||||
var result = await scriptExecutor.ExecuteAsync(
|
||||
inboundMethod,
|
||||
new Dictionary<string, object?>(),
|
||||
route,
|
||||
TimeSpan.FromSeconds(30),
|
||||
ctx.RequestAborted,
|
||||
parentExecutionId: inboundExecutionId);
|
||||
|
||||
ctx.Response.StatusCode = result.Success ? 200 : 500;
|
||||
await ctx.Response.WriteAsync(result.Success ? "ok" : "fail");
|
||||
});
|
||||
|
||||
var client = host.GetTestClient();
|
||||
var response = await client.PostAsync(
|
||||
"/api/submitOrder",
|
||||
new StringContent("{}", Encoding.UTF8, "application/json"));
|
||||
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
|
||||
|
||||
// The routed run emits its sync-ApiCall and NotifySend audit rows on a
|
||||
// deliberately fire-and-forget path (alog.md §7 — an audit write must
|
||||
// never block the user-facing script call). `Notify.Send` therefore
|
||||
// returns — and the routed `RouteToCallAsync` completes — BEFORE the
|
||||
// SqliteAuditWriter background loop has flushed the NotifySend row into
|
||||
// the site hot-path. Wait for all five site rows to be durably present
|
||||
// in SQLite before the central assertion: this is the production
|
||||
// durability point (the row IS in SQLite before it is considered
|
||||
// audited), and pinning it removes the emit-vs-drain race that
|
||||
// otherwise let the SiteAuditTelemetryADrain forward only four rows on
|
||||
// its first tick and leave NotifySend stranded for a full drain
|
||||
// interval under heavy parallel load.
|
||||
await WaitForSiteRowsPersistedAsync(sqliteWriter);
|
||||
|
||||
// The routed run produced a NotifySend that buffered a NotificationSubmit
|
||||
// into S&F. Drain that genuine site-produced submission to the central
|
||||
// NotificationOutboxActor so the NotifyDeliver dispatch rows materialise.
|
||||
await ForwardBufferedNotificationToCentralAsync(
|
||||
storeAndForward, router.NotificationId!, centralProvider, centralAuditWriter);
|
||||
|
||||
// ── Assert ──────────────────────────────────────────────────────────
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var repo = new AuditLogRepository(readContext);
|
||||
|
||||
// Every audit row this site produced (sync ApiCall + cached lifecycle
|
||||
// + NotifySend) plus the central NotifyDeliver rows.
|
||||
var siteRows = await repo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 100));
|
||||
|
||||
// sync ApiCall (1) + cached Submit/Attempted/Resolve (3) + NotifySend (1)
|
||||
// + NotifyDeliver Attempted/Delivered (2) = 7 rows for the routed run.
|
||||
Assert.True(siteRows.Count == 7,
|
||||
"Expected 7 routed-run audit rows; saw: "
|
||||
+ string.Join(", ", siteRows.Select(r => $"{r.Channel}/{r.Kind}/{r.Status}")));
|
||||
Assert.Single(siteRows, r => r.Channel == AuditChannel.ApiOutbound && r.Kind == AuditKind.ApiCall);
|
||||
Assert.Single(siteRows, r => r.Kind == AuditKind.CachedSubmit);
|
||||
Assert.Single(siteRows, r => r.Kind == AuditKind.CachedResolve);
|
||||
Assert.Single(siteRows, r => r.Kind == AuditKind.NotifySend);
|
||||
Assert.Equal(2, siteRows.Count(r => r.Kind == AuditKind.NotifyDeliver));
|
||||
|
||||
// CORE PROMISE: every routed-run row carries the SAME non-null
|
||||
// ParentExecutionId — the inbound request's ExecutionId.
|
||||
var parentIds = siteRows.Select(r => r.ParentExecutionId).Distinct().ToList();
|
||||
Assert.Single(parentIds);
|
||||
Assert.NotNull(parentIds[0]);
|
||||
var inboundExecutionId = parentIds[0]!.Value;
|
||||
|
||||
// The routed run has its OWN distinct ExecutionId — not the parent's.
|
||||
var routedExecutionIds = siteRows
|
||||
.Select(r => r.ExecutionId)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
Assert.Single(routedExecutionIds);
|
||||
Assert.NotNull(routedExecutionIds[0]);
|
||||
var routedExecutionId = routedExecutionIds[0]!.Value;
|
||||
Assert.NotEqual(inboundExecutionId, routedExecutionId);
|
||||
|
||||
// The inbound request's own InboundRequest row is TOP-LEVEL —
|
||||
// ExecutionId = the propagated id, ParentExecutionId = NULL.
|
||||
var inboundRows = await repo.QueryAsync(
|
||||
new AuditLogQueryFilter(ExecutionId: inboundExecutionId),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
var inboundRow = Assert.Single(inboundRows,
|
||||
r => r.Channel == AuditChannel.ApiInbound && r.Kind == AuditKind.InboundRequest);
|
||||
Assert.Equal(AuditStatus.Delivered, inboundRow.Status);
|
||||
Assert.Null(inboundRow.ParentExecutionId);
|
||||
|
||||
// The parentExecutionId filter pulls the routed run's complete
|
||||
// trust-boundary footprint (all 7 routed rows, none of the inbound).
|
||||
var byParent = await repo.QueryAsync(
|
||||
new AuditLogQueryFilter(ParentExecutionId: inboundExecutionId),
|
||||
new AuditLogPaging(PageSize: 100));
|
||||
Assert.Equal(7, byParent.Count);
|
||||
Assert.All(byParent, r => Assert.Equal(routedExecutionId, r.ExecutionId));
|
||||
|
||||
// GetExecutionTreeAsync returns BOTH executions in one chain —
|
||||
// inbound (root) and routed (child), regardless of entry point.
|
||||
var treeFromChild = await repo.GetExecutionTreeAsync(routedExecutionId);
|
||||
AssertChain(treeFromChild, inboundExecutionId, routedExecutionId);
|
||||
var treeFromRoot = await repo.GetExecutionTreeAsync(inboundExecutionId);
|
||||
AssertChain(treeFromRoot, inboundExecutionId, routedExecutionId);
|
||||
}, TimeSpan.FromSeconds(90));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asserts the execution tree is the expected two-node inbound→routed chain:
|
||||
/// the inbound execution is the root (<c>ParentExecutionId = NULL</c>) and the
|
||||
/// routed execution's <c>ParentExecutionId</c> points back at it.
|
||||
/// </summary>
|
||||
private static void AssertChain(
|
||||
IReadOnlyList<ExecutionTreeNode> tree,
|
||||
Guid inboundExecutionId,
|
||||
Guid routedExecutionId)
|
||||
{
|
||||
Assert.Equal(2, tree.Count);
|
||||
var root = Assert.Single(tree, n => n.ExecutionId == inboundExecutionId);
|
||||
Assert.Null(root.ParentExecutionId);
|
||||
var child = Assert.Single(tree, n => n.ExecutionId == routedExecutionId);
|
||||
Assert.Equal(inboundExecutionId, child.ParentExecutionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Spins up a minimal in-memory ASP.NET host whose pipeline mirrors the
|
||||
/// production inbound-API arrangement: routing → the real
|
||||
/// <see cref="AuditWriteMiddleware"/> → the <c>POST /api/{methodName}</c>
|
||||
/// endpoint. The middleware mints + stashes the inbound request's
|
||||
/// <c>ExecutionId</c> and emits the top-level <see cref="AuditKind.InboundRequest"/>
|
||||
/// row via the supplied <see cref="ICentralAuditWriter"/>.
|
||||
/// </summary>
|
||||
private static async Task<IHost> BuildInboundHostAsync(
|
||||
ICentralAuditWriter centralAuditWriter,
|
||||
RequestDelegate endpointHandler)
|
||||
{
|
||||
var hostBuilder = new HostBuilder()
|
||||
.ConfigureWebHost(webBuilder =>
|
||||
{
|
||||
webBuilder
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddSingleton(centralAuditWriter);
|
||||
services.AddRouting();
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseAuditWriteMiddleware();
|
||||
app.UseEndpoints(endpoints =>
|
||||
{
|
||||
endpoints.MapPost("/api/{methodName}", endpointHandler);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
return await hostBuilder.StartAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the genuine site-produced <see cref="NotificationSubmit"/> the routed
|
||||
/// <c>Notify.Send</c> buffered into Store-and-Forward, then drives it through
|
||||
/// a real central <see cref="NotificationOutboxActor"/> so the
|
||||
/// <see cref="AuditKind.NotifyDeliver"/> dispatch rows materialise. The
|
||||
/// dispatcher echoes <c>OriginParentExecutionId</c> off the
|
||||
/// <c>NotificationSubmit</c> onto every <c>NotifyDeliver</c> row — the
|
||||
/// cross-execution linkage under test on the central side.
|
||||
/// </summary>
|
||||
private async Task ForwardBufferedNotificationToCentralAsync(
|
||||
StoreAndForwardService storeAndForward,
|
||||
string notificationId,
|
||||
IServiceProvider centralProvider,
|
||||
ICentralAuditWriter centralAuditWriter)
|
||||
{
|
||||
var buffered = await storeAndForward.GetMessageByIdAsync(notificationId);
|
||||
Assert.NotNull(buffered);
|
||||
var submit = JsonSerializer.Deserialize<NotificationSubmit>(buffered!.PayloadJson);
|
||||
Assert.NotNull(submit);
|
||||
// The routed Notify.Send stamped the inbound request's ExecutionId as the
|
||||
// submission's OriginParentExecutionId — proven separately on the
|
||||
// NotifyDeliver rows, but asserted here too as the central handoff input.
|
||||
Assert.NotNull(submit!.OriginParentExecutionId);
|
||||
|
||||
// The outbox actor runs over the long-lived central provider (which
|
||||
// carries the AlwaysDeliversAdapter) so the dispatch sweep — launched
|
||||
// asynchronously by the DispatchTick — still has a live IServiceProvider
|
||||
// to resolve its per-sweep scope from.
|
||||
var outboxActor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
||||
centralProvider,
|
||||
new NotificationOutboxOptions
|
||||
{
|
||||
// Long timers so PreStart's scheduled ticks never fire — the
|
||||
// test drives ingest + dispatch explicitly.
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromDays(1),
|
||||
},
|
||||
centralAuditWriter,
|
||||
NullLogger<NotificationOutboxActor>.Instance)));
|
||||
|
||||
// Ingest the genuine site submission, then run one dispatch sweep.
|
||||
var ack = await outboxActor.Ask<NotificationSubmitAck>(
|
||||
submit, TimeSpan.FromSeconds(15));
|
||||
Assert.True(ack.Accepted, ack.Error);
|
||||
outboxActor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Polls the site SQLite hot-path until every audit <see cref="AuditKind"/>
|
||||
/// the routed run is expected to emit — sync <c>ApiCall</c>, the cached
|
||||
/// <c>CachedSubmit</c>/<c>ApiCallCached</c>/<c>CachedResolve</c> lifecycle,
|
||||
/// and <c>NotifySend</c> — is durably present (Pending or Forwarded).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The routed run's sync-<c>ApiCall</c> and <c>NotifySend</c> audit rows are
|
||||
/// written fire-and-forget (the script call must not block on the audit
|
||||
/// writer — alog.md §7), so the routed <c>RouteToCallAsync</c> returns
|
||||
/// before the background writer loop has committed those rows.
|
||||
/// <c>NotifySend</c> is emitted last and therefore settles last. This wait
|
||||
/// asserts the specific <b>Kinds</b> are present, not merely a row count: a
|
||||
/// bare count could be satisfied while the last-emitted <c>NotifySend</c>
|
||||
/// row was still in flight, letting the <c>SiteAuditTelemetryActor</c> drain
|
||||
/// only a partial snapshot and leave <c>NotifySend</c> stranded for a later
|
||||
/// tick — the emit-vs-drain race that failed this test under full-suite load.
|
||||
/// </remarks>
|
||||
private async Task WaitForSiteRowsPersistedAsync(SqliteAuditWriter sqliteWriter)
|
||||
{
|
||||
var expectedKinds = new[]
|
||||
{
|
||||
AuditKind.ApiCall, AuditKind.CachedSubmit, AuditKind.ApiCallCached,
|
||||
AuditKind.CachedResolve, AuditKind.NotifySend,
|
||||
};
|
||||
await AwaitAssertAsync(
|
||||
async () =>
|
||||
{
|
||||
var pending = await sqliteWriter.ReadPendingAsync(256);
|
||||
// AuditLog-001: ReadPendingAsync now excludes the cached-lifecycle
|
||||
// kinds (they ride the combined-telemetry drain), so we union
|
||||
// them in via the dedicated read surface to keep the durability
|
||||
// assertion covering EVERY expected Kind.
|
||||
var pendingCached = await sqliteWriter.ReadPendingCachedTelemetryAsync(256);
|
||||
var forwarded = await sqliteWriter.ReadForwardedAsync(256);
|
||||
var kinds = pending.Concat(pendingCached).Concat(forwarded)
|
||||
.Select(r => r.Kind).ToHashSet();
|
||||
var missing = expectedKinds.Where(k => !kinds.Contains(k)).ToList();
|
||||
Assert.True(
|
||||
missing.Count == 0,
|
||||
"Expected every routed-run audit Kind durably in SQLite; missing: "
|
||||
+ string.Join(", ", missing)
|
||||
+ $" (saw {pending.Count} Pending + {pendingCached.Count} PendingCached + {forwarded.Count} Forwarded).");
|
||||
},
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stub <see cref="INotificationDeliveryAdapter"/> that always reports a
|
||||
/// successful delivery — a single dispatch sweep then yields one
|
||||
/// <see cref="AuditStatus.Attempted"/> + one <see cref="AuditStatus.Delivered"/>
|
||||
/// <see cref="AuditKind.NotifyDeliver"/> row.
|
||||
/// </summary>
|
||||
private sealed class AlwaysDeliversAdapter : INotificationDeliveryAdapter
|
||||
{
|
||||
public NotificationType Type => NotificationType.Email;
|
||||
|
||||
public Task<DeliveryOutcome> DeliverAsync(
|
||||
ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications.Notification notification,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(DeliveryOutcome.Success("ops@example.com"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-process stand-in for the cross-cluster routing transport
|
||||
/// (<c>CommunicationServiceInstanceRouter</c> →
|
||||
/// <c>CommunicationService</c> → site <c>DeploymentManagerActor</c>). On a
|
||||
/// routed <c>Call</c> it does exactly what the production site handler does:
|
||||
/// it reads <see cref="RouteToCallRequest.ParentExecutionId"/> off the wire
|
||||
/// request and threads it into a fresh routed <see cref="ScriptRuntimeContext"/>
|
||||
/// as <c>parentExecutionId</c>, then runs the routed script's three
|
||||
/// trust-boundary actions (sync <c>ExternalSystem.Call</c>, a cached call and
|
||||
/// a <c>Notify.Send</c>). The routed context still mints its OWN fresh
|
||||
/// <c>ExecutionId</c> — only the parent pointer is inherited.
|
||||
/// </summary>
|
||||
private sealed class BridgingInstanceRouter : IInstanceRouter
|
||||
{
|
||||
private readonly string _siteId;
|
||||
private readonly IExternalSystemClient _externalClient;
|
||||
private readonly IAuditWriter _auditWriter;
|
||||
private readonly ICachedCallTelemetryForwarder _cachedForwarder;
|
||||
private readonly StoreAndForwardService _storeAndForward;
|
||||
|
||||
/// <summary>
|
||||
/// The <c>NotificationId</c> the routed <c>Notify.Send</c> minted, captured
|
||||
/// so the test can drain the buffered <see cref="NotificationSubmit"/>.
|
||||
/// </summary>
|
||||
public string? NotificationId { get; private set; }
|
||||
|
||||
public BridgingInstanceRouter(
|
||||
string siteId,
|
||||
IExternalSystemClient externalClient,
|
||||
IAuditWriter auditWriter,
|
||||
ICachedCallTelemetryForwarder cachedForwarder,
|
||||
StoreAndForwardService storeAndForward)
|
||||
{
|
||||
_siteId = siteId;
|
||||
_externalClient = externalClient;
|
||||
_auditWriter = auditWriter;
|
||||
_cachedForwarder = cachedForwarder;
|
||||
_storeAndForward = storeAndForward;
|
||||
}
|
||||
|
||||
public async Task<RouteToCallResponse> RouteToCallAsync(
|
||||
string siteId, RouteToCallRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
var sharedScriptLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
|
||||
// Mirror DeploymentManagerActor → ScriptActor → ScriptExecutionActor:
|
||||
// the routed script execution gets its OWN fresh ExecutionId, and the
|
||||
// inbound request's ExecutionId arrives as ParentExecutionId.
|
||||
var routedContext = new ScriptRuntimeContext(
|
||||
ActorRefs.Nobody,
|
||||
ActorRefs.Nobody,
|
||||
sharedScriptLibrary,
|
||||
currentCallDepth: 0,
|
||||
maxCallDepth: 10,
|
||||
askTimeout: TimeSpan.FromSeconds(5),
|
||||
instanceName: request.InstanceUniqueName,
|
||||
logger: NullLogger.Instance,
|
||||
externalSystemClient: _externalClient,
|
||||
databaseGateway: null,
|
||||
storeAndForward: _storeAndForward,
|
||||
siteCommunicationActor: null,
|
||||
siteId: _siteId,
|
||||
sourceScript: $"ScriptActor:{request.ScriptName}",
|
||||
auditWriter: _auditWriter,
|
||||
operationTrackingStore: null,
|
||||
cachedForwarder: _cachedForwarder,
|
||||
executionId: null,
|
||||
parentExecutionId: request.ParentExecutionId);
|
||||
|
||||
// The routed site script's body: a sync ExternalSystem.Call, a cached
|
||||
// call, and a Notify.Send — three distinct trust-boundary actions of
|
||||
// the one routed execution.
|
||||
await routedContext.ExternalSystem.Call(ExternalSystemName, ExternalMethodName);
|
||||
await routedContext.ExternalSystem.CachedCall(ExternalSystemName, ExternalMethodName);
|
||||
NotificationId = await routedContext.Notify
|
||||
.To(NotifyListName)
|
||||
.Send("Routed run alert", "inbound-routed script fired");
|
||||
|
||||
return new RouteToCallResponse(
|
||||
request.CorrelationId, true, "routed-ok", null, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
public Task<RouteToGetAttributesResponse> RouteToGetAttributesAsync(
|
||||
string siteId, RouteToGetAttributesRequest request, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<RouteToSetAttributesResponse> RouteToSetAttributesAsync(
|
||||
string siteId, RouteToSetAttributesRequest request, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle F (#23 M6-T12) end-to-end tests for the
|
||||
/// <see cref="AuditLogPartitionMaintenanceService"/> hosted service running
|
||||
/// the real EF/MSSQL <see cref="AuditLogPartitionMaintenance"/> against the
|
||||
/// per-class <see cref="MsSqlMigrationFixture"/>. The migration seeds
|
||||
/// boundaries for every month Jan 2026 – Dec 2027, so the eager startup tick
|
||||
/// can be exercised both for the "future covered" no-op case and for the
|
||||
/// "lookahead larger than covered" SPLIT case.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Tests within this class share one fixture DB — boundaries added by one
|
||||
/// test persist across the next. Each test reads the max boundary at the
|
||||
/// start and computes its lookahead relative to it, mirroring the pattern
|
||||
/// used by the per-component <c>AuditLogPartitionMaintenanceTests</c> in
|
||||
/// <c>ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests</c>.
|
||||
/// </remarks>
|
||||
public class PartitionMaintenanceTests : IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public PartitionMaintenanceTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext CreateContext() =>
|
||||
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString).Options);
|
||||
|
||||
/// <summary>
|
||||
/// Builds the central-side DI graph for the hosted service: scoped EF
|
||||
/// context + scoped <see cref="IPartitionMaintenance"/> matching how
|
||||
/// <c>AddConfigurationDatabase</c> wires the production composition root.
|
||||
/// </summary>
|
||||
private ServiceProvider BuildProvider()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(
|
||||
opts => opts.UseSqlServer(_fixture.ConnectionString),
|
||||
ServiceLifetime.Scoped);
|
||||
services.AddScoped<IPartitionMaintenance, AuditLogPartitionMaintenance>();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static async Task<DateTime?> ReadMaxBoundaryAsync(IServiceProvider sp)
|
||||
{
|
||||
await using var scope = sp.CreateAsyncScope();
|
||||
var maintenance = scope.ServiceProvider.GetRequiredService<IPartitionMaintenance>();
|
||||
return await maintenance.GetMaxBoundaryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mirrors the helper in
|
||||
/// <c>AuditLogPartitionMaintenanceTests.LookaheadForExtraBoundaries</c>:
|
||||
/// the smallest lookahead value that lands the SPLIT horizon exactly
|
||||
/// <paramref name="extraBoundaries"/> months past the current max.
|
||||
/// </summary>
|
||||
private static int LookaheadForExtraBoundaries(DateTime max, int extraBoundaries)
|
||||
{
|
||||
var nowFirstOfNextMonth = FirstOfNextMonth(DateTime.UtcNow);
|
||||
var monthsToMax = ((max.Year - nowFirstOfNextMonth.Year) * 12)
|
||||
+ max.Month - nowFirstOfNextMonth.Month;
|
||||
return monthsToMax + extraBoundaries;
|
||||
}
|
||||
|
||||
private static int LookaheadInsideExistingRange(DateTime max)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var months = ((max.Year - now.Year) * 12) + max.Month - now.Month - 1;
|
||||
return Math.Max(1, months);
|
||||
}
|
||||
|
||||
private static DateTime FirstOfNextMonth(DateTime instant)
|
||||
{
|
||||
var firstOfThisMonth = new DateTime(instant.Year, instant.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
return firstOfThisMonth.AddMonths(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Awaits one full tick of the hosted service. The service runs an
|
||||
/// eager startup tick inside <see cref="AuditLogPartitionMaintenanceService.StartAsync"/>'s
|
||||
/// continuation, but the continuation is dispatched on a background
|
||||
/// Task.Run — so we poll the side effect (the boundary count or
|
||||
/// max-boundary value) until it changes.
|
||||
/// </summary>
|
||||
private async Task StartAndAwaitStartupTickAsync(
|
||||
AuditLogPartitionMaintenanceService svc,
|
||||
Func<Task<bool>> awaitCondition,
|
||||
TimeSpan timeout)
|
||||
{
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (await awaitCondition())
|
||||
{
|
||||
return;
|
||||
}
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 1. EndToEnd_DefaultLookahead_NoSplit_WhenFutureCovered
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_DefaultLookahead_NoSplit_WhenFutureCovered()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
await using var sp = BuildProvider();
|
||||
|
||||
// The migration seeds boundaries through Dec 2027. With default
|
||||
// lookahead = 1 and today = ~2026-05-20, horizon =
|
||||
// NormalizeToFirstOfMonth(now) + 1 = 2026-07-01, well within the
|
||||
// seeded range, so the startup tick should issue zero SPLITs.
|
||||
var maxBefore = await ReadMaxBoundaryAsync(sp);
|
||||
Assert.NotNull(maxBefore);
|
||||
|
||||
// Skip if the fixture DB already has boundaries past Dec 2027 from
|
||||
// a prior test in this class — the lookahead-already-covered path
|
||||
// is what we want to exercise, regardless of how far past Dec 2027
|
||||
// the boundary may be.
|
||||
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
|
||||
{
|
||||
IntervalSeconds = 60, // long enough that only the startup tick fires inside the test window
|
||||
LookaheadMonths = 1,
|
||||
});
|
||||
|
||||
var svc = new AuditLogPartitionMaintenanceService(
|
||||
sp.GetRequiredService<IServiceScopeFactory>(),
|
||||
opts,
|
||||
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
|
||||
|
||||
// Drive the startup tick. There is no public completion handle;
|
||||
// poll until either (a) the max boundary changes (which would be a
|
||||
// failure for this test) or (b) the polling window expires (success).
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
svc.Dispose();
|
||||
|
||||
// Assert the max boundary is unchanged: no SPLIT was issued.
|
||||
var maxAfter = await ReadMaxBoundaryAsync(sp);
|
||||
Assert.Equal(maxBefore, maxAfter);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 2. EndToEnd_LookaheadLargerThanCovered_Splits_NewBoundaries
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_LookaheadLargerThanCovered_Splits_NewBoundaries()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
await using var sp = BuildProvider();
|
||||
|
||||
var maxBefore = await ReadMaxBoundaryAsync(sp);
|
||||
Assert.NotNull(maxBefore);
|
||||
|
||||
// Pick a lookahead that adds exactly two new boundaries past the
|
||||
// current max. The expected new boundaries are max+1mo and max+2mo.
|
||||
var lookahead = LookaheadForExtraBoundaries(maxBefore.Value, extraBoundaries: 2);
|
||||
var expectedFirstNew = maxBefore.Value.AddMonths(1);
|
||||
var expectedSecondNew = maxBefore.Value.AddMonths(2);
|
||||
|
||||
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
|
||||
{
|
||||
IntervalSeconds = 60,
|
||||
LookaheadMonths = lookahead,
|
||||
});
|
||||
|
||||
var svc = new AuditLogPartitionMaintenanceService(
|
||||
sp.GetRequiredService<IServiceScopeFactory>(),
|
||||
opts,
|
||||
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
|
||||
|
||||
// Drive the startup tick. Wait until max boundary moves forward by
|
||||
// the expected amount; SPLIT against MSSQL can take a second or two
|
||||
// on a busy dev container.
|
||||
await StartAndAwaitStartupTickAsync(
|
||||
svc,
|
||||
async () =>
|
||||
{
|
||||
var current = await ReadMaxBoundaryAsync(sp);
|
||||
return current == expectedSecondNew;
|
||||
},
|
||||
timeout: TimeSpan.FromSeconds(15));
|
||||
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
svc.Dispose();
|
||||
|
||||
var maxAfter = await ReadMaxBoundaryAsync(sp);
|
||||
// Two new boundaries should be present after the startup tick. The
|
||||
// hosted service does not surface the added-list directly (it logs
|
||||
// only at Information), so we assert via the max-boundary delta.
|
||||
Assert.Equal(expectedSecondNew, maxAfter);
|
||||
// Sanity: the intermediate boundary was also added (the loop
|
||||
// SPLITs every month from max+1 up to horizon, in order).
|
||||
Assert.True(expectedFirstNew < expectedSecondNew);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 3. EndToEnd_PartitionMaintenance_Idempotent_OverTwoRuns
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_PartitionMaintenance_Idempotent_OverTwoRuns()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
await using var sp = BuildProvider();
|
||||
|
||||
var maxBefore = await ReadMaxBoundaryAsync(sp);
|
||||
Assert.NotNull(maxBefore);
|
||||
|
||||
// Add exactly one new boundary on the first run.
|
||||
var lookahead = LookaheadForExtraBoundaries(maxBefore.Value, extraBoundaries: 1);
|
||||
var expectedAdded = maxBefore.Value.AddMonths(1);
|
||||
|
||||
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
|
||||
{
|
||||
IntervalSeconds = 60,
|
||||
LookaheadMonths = lookahead,
|
||||
});
|
||||
|
||||
// First run.
|
||||
var svc1 = new AuditLogPartitionMaintenanceService(
|
||||
sp.GetRequiredService<IServiceScopeFactory>(),
|
||||
opts,
|
||||
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
|
||||
await StartAndAwaitStartupTickAsync(
|
||||
svc1,
|
||||
async () =>
|
||||
{
|
||||
var current = await ReadMaxBoundaryAsync(sp);
|
||||
return current == expectedAdded;
|
||||
},
|
||||
timeout: TimeSpan.FromSeconds(15));
|
||||
await svc1.StopAsync(CancellationToken.None);
|
||||
svc1.Dispose();
|
||||
|
||||
var maxAfterFirst = await ReadMaxBoundaryAsync(sp);
|
||||
Assert.Equal(expectedAdded, maxAfterFirst);
|
||||
|
||||
// Second run with the SAME lookahead value. Because the boundary
|
||||
// is already covered, the EnsureLookaheadAsync call must be a
|
||||
// no-op — max boundary is unchanged AND no exception is thrown.
|
||||
var svc2 = new AuditLogPartitionMaintenanceService(
|
||||
sp.GetRequiredService<IServiceScopeFactory>(),
|
||||
opts,
|
||||
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
|
||||
await svc2.StartAsync(CancellationToken.None);
|
||||
// Wait long enough that the startup tick would have fired and
|
||||
// logged any boundary addition; the boundary state must remain
|
||||
// unchanged after the wait.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
await svc2.StopAsync(CancellationToken.None);
|
||||
svc2.Dispose();
|
||||
|
||||
var maxAfterSecond = await ReadMaxBoundaryAsync(sp);
|
||||
Assert.Equal(maxAfterFirst, maxAfterSecond);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle F (#23 M6-T11) end-to-end test for the daily partition-switch
|
||||
/// purge: seeds three monthly partitions (Jan / Feb / Mar 2026) with direct
|
||||
/// INSERTs that bypass the standard repository ingest path (so the seed
|
||||
/// timestamps are explicit), drives <see cref="AuditLogPurgeActor"/> against
|
||||
/// the real <see cref="AuditLogRepository"/> + per-test
|
||||
/// <see cref="MsSqlMigrationFixture"/> database, and asserts:
|
||||
/// <list type="number">
|
||||
/// <item>The oldest partition (Jan) is removed.</item>
|
||||
/// <item>Newer partitions (Feb + Mar) are untouched.</item>
|
||||
/// <item>The <c>UX_AuditLog_EventId</c> unique index survives the
|
||||
/// drop-and-rebuild dance.</item>
|
||||
/// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains
|
||||
/// idempotent against the rebuilt index after the purge.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The brief calls out that direct INSERTs bypass the writer role's INSERT-only
|
||||
/// grant; the fixture connects as <c>sa</c> (see
|
||||
/// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so
|
||||
/// the seed step does not need the writer role at all. The drop-and-rebuild
|
||||
/// dance itself runs under the same admin connection because the test owns
|
||||
/// the database — the role granularity is exercised in the repository tests,
|
||||
/// not here.
|
||||
/// </remarks>
|
||||
public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public PartitionPurgeTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext CreateContext() =>
|
||||
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString).Options);
|
||||
|
||||
/// <summary>
|
||||
/// Direct INSERT into <c>dbo.AuditLog</c> bypassing
|
||||
/// <see cref="IAuditLogRepository.InsertIfNotExistsAsync"/>. Used by the
|
||||
/// seed step so the test can place rows in arbitrary partitions without
|
||||
/// the repository's idempotency wrapper or ingest-stamping behaviour
|
||||
/// affecting the seed payload.
|
||||
/// </summary>
|
||||
private async Task DirectInsertAsync(
|
||||
SqlConnection conn,
|
||||
Guid eventId,
|
||||
DateTime occurredAtUtc,
|
||||
string siteId)
|
||||
{
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
INSERT INTO dbo.AuditLog
|
||||
(EventId, OccurredAtUtc, IngestedAtUtc, Channel, Kind, CorrelationId,
|
||||
SourceSiteId, SourceInstanceId, SourceScript, Actor, Target, Status,
|
||||
HttpStatus, DurationMs, ErrorMessage, ErrorDetail, RequestSummary,
|
||||
ResponseSummary, PayloadTruncated, Extra, ForwardState)
|
||||
VALUES
|
||||
(@EventId, @OccurredAtUtc, @IngestedAtUtc, 'ApiOutbound', 'ApiCall', NULL,
|
||||
@SourceSiteId, NULL, NULL, NULL, NULL, 'Delivered',
|
||||
NULL, NULL, NULL, NULL, NULL,
|
||||
NULL, 0, NULL, NULL);";
|
||||
cmd.Parameters.Add("@EventId", System.Data.SqlDbType.UniqueIdentifier).Value = eventId;
|
||||
// SqlDbType.DateTime2 with explicit Scale 7 matches the
|
||||
// OccurredAtUtc column shape (datetime2(7)) and avoids the implicit
|
||||
// narrowing that SqlClient's default DateTime → datetime applies via
|
||||
// AddWithValue. Critical for partition assignment: the partition
|
||||
// function key column is datetime2(7); a narrowed value would still
|
||||
// land in the correct partition for first-of-month seeds, but
|
||||
// explicit typing here documents the intent and matches how the
|
||||
// production repository INSERT shapes its parameters.
|
||||
var occurredParam = cmd.Parameters.Add("@OccurredAtUtc", System.Data.SqlDbType.DateTime2);
|
||||
occurredParam.Scale = 7;
|
||||
occurredParam.Value = occurredAtUtc;
|
||||
var ingestedParam = cmd.Parameters.Add("@IngestedAtUtc", System.Data.SqlDbType.DateTime2);
|
||||
ingestedParam.Scale = 7;
|
||||
ingestedParam.Value = DateTime.UtcNow;
|
||||
cmd.Parameters.Add("@SourceSiteId", System.Data.SqlDbType.VarChar, 64).Value = siteId;
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asserts that <c>UX_AuditLog_EventId</c> exists in
|
||||
/// <c>sys.indexes</c>. The drop-and-rebuild dance briefly removes the
|
||||
/// index inside its transaction; this check is meant to fire AFTER the
|
||||
/// actor's purge tick has committed so the rebuilt index is observable.
|
||||
/// </summary>
|
||||
private static async Task AssertUxIndexExistsAsync(SqlConnection conn)
|
||||
{
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
SELECT COUNT(*)
|
||||
FROM sys.indexes
|
||||
WHERE name = 'UX_AuditLog_EventId'
|
||||
AND object_id = OBJECT_ID('dbo.AuditLog');";
|
||||
var raw = await cmd.ExecuteScalarAsync();
|
||||
var count = Convert.ToInt32(raw);
|
||||
Assert.True(count == 1, $"UX_AuditLog_EventId should be present post-purge; sys.indexes count was {count}.");
|
||||
}
|
||||
|
||||
private IActorRef CreateActor(
|
||||
IServiceProvider sp,
|
||||
AuditLogPurgeOptions purgeOptions,
|
||||
AuditLogOptions auditOptions)
|
||||
{
|
||||
return Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
|
||||
sp,
|
||||
Options.Create(purgeOptions),
|
||||
Options.Create(auditOptions),
|
||||
NullLogger<AuditLogPurgeActor>.Instance)));
|
||||
}
|
||||
|
||||
private static (DateTime Jan, DateTime Feb, DateTime Mar) SeedOccurredAt() => (
|
||||
new DateTime(2026, 1, 15, 0, 0, 0, DateTimeKind.Utc),
|
||||
new DateTime(2026, 2, 15, 0, 0, 0, DateTimeKind.Utc),
|
||||
new DateTime(2026, 3, 15, 0, 0, 0, DateTimeKind.Utc));
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 1. EndToEnd_OldestPartition_PurgedViaActor_NewerKept
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_OldestPartition_PurgedViaActor_NewerKept()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
// Test date is ~2026-05-20 per environment. We want a threshold that
|
||||
// sits strictly between Jan 15 (the Jan partition's MAX) and Feb 15
|
||||
// (the Feb partition's MAX) so only the Jan-2026 partition is
|
||||
// eligible for purge. RetentionDays = 100 gives a threshold of
|
||||
// ~2026-02-09 — Jan 15 is older (purged), Feb 15 and Mar 15 are
|
||||
// newer (kept). The window between Jan 15 and Feb 15 is wide enough
|
||||
// (~30 days) to tolerate any plausible test-clock drift in CI.
|
||||
var siteId = "purge-e2e-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
var janEventId = Guid.NewGuid();
|
||||
var febEventId = Guid.NewGuid();
|
||||
var marEventId = Guid.NewGuid();
|
||||
var (janOccurred, febOccurred, marOccurred) = SeedOccurredAt();
|
||||
|
||||
await using (var seedConn = _fixture.OpenConnection())
|
||||
{
|
||||
await DirectInsertAsync(seedConn, janEventId, janOccurred, siteId);
|
||||
await DirectInsertAsync(seedConn, febEventId, febOccurred, siteId);
|
||||
await DirectInsertAsync(seedConn, marEventId, marOccurred, siteId);
|
||||
}
|
||||
|
||||
// Wire the actor with a real EF context against the fixture DB.
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(
|
||||
opts => opts.UseSqlServer(_fixture.ConnectionString),
|
||||
ServiceLifetime.Scoped);
|
||||
services.AddScoped<IAuditLogRepository, AuditLogRepository>();
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var probe = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent));
|
||||
|
||||
var purgeOptions = new AuditLogPurgeOptions
|
||||
{
|
||||
IntervalHours = 24,
|
||||
IntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||
};
|
||||
var auditOptions = new AuditLogOptions { RetentionDays = 100 };
|
||||
|
||||
CreateActor(sp, purgeOptions, auditOptions);
|
||||
|
||||
// Wait for the actor's tick to purge the Jan-2026 partition.
|
||||
// Concurrent test runs against the same fixture might also create
|
||||
// eligible partitions, but each test class owns its own fixture DB
|
||||
// (MsSqlMigrationFixture seeds a guid-named DB per class), so the
|
||||
// Jan-2026 boundary is the only one this test can have produced.
|
||||
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var matched = probe.FishForMessage<AuditLogPurgedEvent>(
|
||||
isMessage: m => m.MonthBoundary == janBoundary,
|
||||
max: TimeSpan.FromSeconds(30));
|
||||
Assert.True(matched.RowsDeleted >= 1,
|
||||
$"Expected RowsDeleted >= 1 for Jan-2026 boundary; got {matched.RowsDeleted}.");
|
||||
|
||||
// Allow a brief settle in case the actor is mid-tick on Feb/Mar
|
||||
// (it shouldn't be, since RetentionDays = 90 means only Jan is
|
||||
// eligible, but the actor MAY re-enumerate quickly while we read).
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
await using var verify = CreateContext();
|
||||
var rows = await verify.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
|
||||
// Jan removed; Feb + Mar untouched. Because the test owns the site
|
||||
// id and the fixture DB, exact set membership is observable.
|
||||
Assert.DoesNotContain(rows, r => r.EventId == janEventId);
|
||||
Assert.Contains(rows, r => r.EventId == febEventId);
|
||||
Assert.Contains(rows, r => r.EventId == marEventId);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 2. EndToEnd_UxIndexRebuilt_AfterPurge
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_UxIndexRebuilt_AfterPurge()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
// Same shape as test 1 — purge the Jan-2026 partition and then
|
||||
// assert the UX_AuditLog_EventId index is still present. The
|
||||
// drop-and-rebuild dance briefly removes it inside its transaction
|
||||
// (the SWITCH PARTITION step requires the non-aligned unique index
|
||||
// to be absent), but step 5 rebuilds it before committing. Sanity-
|
||||
// checking the post-COMMIT shape here documents the invariant in an
|
||||
// assertable way.
|
||||
var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
var janEventId = Guid.NewGuid();
|
||||
var (janOccurred, _, _) = SeedOccurredAt();
|
||||
|
||||
await using (var seedConn = _fixture.OpenConnection())
|
||||
{
|
||||
await DirectInsertAsync(seedConn, janEventId, janOccurred, siteId);
|
||||
}
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(
|
||||
opts => opts.UseSqlServer(_fixture.ConnectionString),
|
||||
ServiceLifetime.Scoped);
|
||||
services.AddScoped<IAuditLogRepository, AuditLogRepository>();
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var probe = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent));
|
||||
|
||||
CreateActor(
|
||||
sp,
|
||||
new AuditLogPurgeOptions
|
||||
{
|
||||
IntervalHours = 24,
|
||||
IntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||
},
|
||||
new AuditLogOptions { RetentionDays = 90 });
|
||||
|
||||
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
probe.FishForMessage<AuditLogPurgedEvent>(
|
||||
isMessage: m => m.MonthBoundary == janBoundary,
|
||||
max: TimeSpan.FromSeconds(30));
|
||||
|
||||
// Open a fresh connection (the actor's pool is owned by EF) and
|
||||
// assert the index is present post-purge.
|
||||
await using var check = _fixture.OpenConnection();
|
||||
await AssertUxIndexExistsAsync(check);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 3. EndToEnd_InsertIfNotExistsAsync_StillIdempotent_AfterPurge
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_InsertIfNotExistsAsync_StillIdempotent_AfterPurge()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
// Seed + purge a Jan-2026 row, THEN exercise InsertIfNotExistsAsync
|
||||
// twice for a fresh (May-2026) EventId. The second call must be a
|
||||
// no-op (duplicate-key collision swallowed by the repository, per
|
||||
// M2 Bundle A's race-fix) — which means the rebuilt
|
||||
// UX_AuditLog_EventId unique index is functioning as intended.
|
||||
var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
var janEventId = Guid.NewGuid();
|
||||
var (janOccurred, _, _) = SeedOccurredAt();
|
||||
|
||||
await using (var seedConn = _fixture.OpenConnection())
|
||||
{
|
||||
await DirectInsertAsync(seedConn, janEventId, janOccurred, siteId);
|
||||
}
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddDbContext<ScadaBridgeDbContext>(
|
||||
opts => opts.UseSqlServer(_fixture.ConnectionString),
|
||||
ServiceLifetime.Scoped);
|
||||
services.AddScoped<IAuditLogRepository, AuditLogRepository>();
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var probe = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent));
|
||||
|
||||
CreateActor(
|
||||
sp,
|
||||
new AuditLogPurgeOptions
|
||||
{
|
||||
IntervalHours = 24,
|
||||
IntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||
},
|
||||
new AuditLogOptions { RetentionDays = 90 });
|
||||
|
||||
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
probe.FishForMessage<AuditLogPurgedEvent>(
|
||||
isMessage: m => m.MonthBoundary == janBoundary,
|
||||
max: TimeSpan.FromSeconds(30));
|
||||
|
||||
// Settle then exercise InsertIfNotExistsAsync twice for the same
|
||||
// EventId. The repository's idempotency relies on
|
||||
// UX_AuditLog_EventId being present so the IF NOT EXISTS … INSERT
|
||||
// race window resolves to a duplicate-key violation the repo
|
||||
// swallows. If the index were missing here, two rows would land
|
||||
// and the second InsertIfNotExistsAsync would silently double-insert.
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
var freshEventId = Guid.NewGuid();
|
||||
var freshOccurred = new DateTime(2026, 5, 15, 12, 0, 0, DateTimeKind.Utc);
|
||||
var freshSite = "purge-idem-fresh-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
var freshEvt = new AuditEvent
|
||||
{
|
||||
EventId = freshEventId,
|
||||
OccurredAtUtc = freshOccurred,
|
||||
Channel = AuditChannel.ApiOutbound,
|
||||
Kind = AuditKind.ApiCall,
|
||||
Status = AuditStatus.Delivered,
|
||||
SourceSiteId = freshSite,
|
||||
Target = "system-x/method",
|
||||
};
|
||||
|
||||
await using (var ctx = CreateContext())
|
||||
{
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
await repo.InsertIfNotExistsAsync(freshEvt);
|
||||
// Same row a second time — must be a silent no-op.
|
||||
await repo.InsertIfNotExistsAsync(freshEvt);
|
||||
}
|
||||
|
||||
await using var verify = CreateContext();
|
||||
var rows = await verify.Set<AuditEvent>()
|
||||
.Where(e => e.SourceSiteId == freshSite)
|
||||
.ToListAsync();
|
||||
Assert.Single(rows);
|
||||
Assert.Equal(freshEventId, rows[0].EventId);
|
||||
}
|
||||
}
|
||||
+272
@@ -0,0 +1,272 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration.Infrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle H — end-to-end test wiring the full Audit Log #23 M2 sync-call pipeline:
|
||||
/// <see cref="FallbackAuditWriter"/> over a <see cref="SqliteAuditWriter"/> backed by
|
||||
/// an in-memory SQLite database; the <see cref="SiteAuditTelemetryActor"/> drains
|
||||
/// Pending rows and pushes them through a stub <see cref="ISiteStreamAuditClient"/>
|
||||
/// that forwards directly to the central <see cref="AuditLogIngestActor"/> backed
|
||||
/// by a real <see cref="AuditLogRepository"/> on the <see cref="MsSqlMigrationFixture"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is a <b>component-level</b> integration test, not a full Akka-cluster
|
||||
/// test (per the M2 brainstorm decision). The stub gRPC client short-circuits
|
||||
/// the wire so we exercise the real telemetry actor, the real ingest actor, the
|
||||
/// real SQLite writer, and the real MSSQL repository — without standing up a
|
||||
/// Kestrel host or two-cluster topology.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The site-side telemetry actor's <c>Drain</c> message is private; rather than
|
||||
/// expose it we drive the drain by setting <c>BusyIntervalSeconds = 1</c> so the
|
||||
/// initial scheduled tick fires within a second of actor start. Tests then
|
||||
/// <see cref="TestKitBase.AwaitAssertAsync"/> until the central repository
|
||||
/// observes the expected rows.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Each test uses a unique <c>SourceSiteId</c> (Guid suffix) so concurrent tests
|
||||
/// and the per-fixture MSSQL database lifetime don't interfere with each other.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class SyncCallEmissionEndToEndTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public SyncCallEmissionEndToEndTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
private static string NewSiteId() =>
|
||||
"test-bundle-h-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
private static AuditEvent NewEvent(string siteId, Guid? id = null) => new()
|
||||
{
|
||||
EventId = id ?? Guid.NewGuid(),
|
||||
OccurredAtUtc = DateTime.UtcNow,
|
||||
Channel = AuditChannel.ApiOutbound,
|
||||
Kind = AuditKind.ApiCall,
|
||||
Status = AuditStatus.Delivered,
|
||||
SourceSiteId = siteId,
|
||||
Target = "external-system-a/method",
|
||||
};
|
||||
|
||||
private static IOptions<SqliteAuditWriterOptions> InMemorySqliteOptions() =>
|
||||
Options.Create(new SqliteAuditWriterOptions
|
||||
{
|
||||
// Per-test unique database name + Mode=Memory + Cache=Shared keeps
|
||||
// the in-memory database alive for the duration of the test even
|
||||
// though Microsoft.Data.Sqlite tears the file down with the last
|
||||
// connection. The DatabasePath field is unused because we override
|
||||
// the connection string below.
|
||||
DatabasePath = "ignored",
|
||||
BatchSize = 64,
|
||||
ChannelCapacity = 1024,
|
||||
});
|
||||
|
||||
private static SqliteAuditWriter CreateInMemorySqliteWriter() =>
|
||||
// The 4th constructor argument is connectionStringOverride. A unique
|
||||
// shared-cache in-memory URI keeps the schema scoped to this writer
|
||||
// instance and torn down when the writer is disposed.
|
||||
new SqliteAuditWriter(
|
||||
InMemorySqliteOptions(),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
connectionStringOverride: $"Data Source=file:auditlog-h-{Guid.NewGuid():N}?mode=memory&cache=shared");
|
||||
|
||||
private static IOptions<SiteAuditTelemetryOptions> FastTelemetryOptions() =>
|
||||
Options.Create(new SiteAuditTelemetryOptions
|
||||
{
|
||||
BatchSize = 256,
|
||||
// 1s for both intervals so the initial scheduled tick fires fast
|
||||
// and any failure-driven re-tick also fires fast — without
|
||||
// requiring a public Drain message to be exposed.
|
||||
BusyIntervalSeconds = 1,
|
||||
IdleIntervalSeconds = 1,
|
||||
});
|
||||
|
||||
private IActorRef CreateIngestActor(IAuditLogRepository repo) =>
|
||||
Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
|
||||
repo,
|
||||
NullLogger<AuditLogIngestActor>.Instance)));
|
||||
|
||||
private IActorRef CreateTelemetryActor(
|
||||
ISiteAuditQueue queue,
|
||||
ISiteStreamAuditClient client) =>
|
||||
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
|
||||
queue,
|
||||
client,
|
||||
FastTelemetryOptions(),
|
||||
NullLogger<SiteAuditTelemetryActor>.Instance)));
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_OneWrittenEvent_Reaches_Central_AuditLog_Within_Reasonable_Time()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
|
||||
// Real central wiring: repo + ingest actor.
|
||||
await using var ingestContext = CreateContext();
|
||||
var ingestRepo = new AuditLogRepository(ingestContext);
|
||||
var ingestActor = CreateIngestActor(ingestRepo);
|
||||
|
||||
// Real site wiring: SQLite (in-memory) + ring + fallback + telemetry.
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var ring = new RingBufferFallback();
|
||||
var fallback = new FallbackAuditWriter(
|
||||
sqliteWriter,
|
||||
ring,
|
||||
new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
|
||||
CreateTelemetryActor(sqliteWriter, stubClient);
|
||||
|
||||
// Act — one fresh event written via the FallbackAuditWriter hot-path.
|
||||
var evt = NewEvent(siteId);
|
||||
await fallback.WriteAsync(evt);
|
||||
|
||||
// Assert — the central AuditLog row materialises within a window that
|
||||
// covers initial tick (1s) + a generous slack for SQLite + the actor
|
||||
// round-trip + EF/MSSQL latency.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var readRepo = new AuditLogRepository(readContext);
|
||||
var rows = await readRepo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
Assert.Single(rows);
|
||||
Assert.Equal(evt.EventId, rows[0].EventId);
|
||||
// Central stamps IngestedAtUtc; site never sets it.
|
||||
Assert.NotNull(rows[0].IngestedAtUtc);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_GrpcStubError_RowStays_Pending_NextTick_Succeeds()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
|
||||
await using var ingestContext = CreateContext();
|
||||
var ingestRepo = new AuditLogRepository(ingestContext);
|
||||
var ingestActor = CreateIngestActor(ingestRepo);
|
||||
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var ring = new RingBufferFallback();
|
||||
var fallback = new FallbackAuditWriter(
|
||||
sqliteWriter,
|
||||
ring,
|
||||
new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
|
||||
// Stub fails the first push; subsequent calls flow through. The
|
||||
// telemetry actor's on-failure branch keeps rows in Pending state, so
|
||||
// the next tick re-reads them and tries again.
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor)
|
||||
{
|
||||
FailNextCallCount = 1,
|
||||
};
|
||||
CreateTelemetryActor(sqliteWriter, stubClient);
|
||||
|
||||
var evt = NewEvent(siteId);
|
||||
await fallback.WriteAsync(evt);
|
||||
|
||||
// Wait long enough for at least one failure-then-success cycle. With
|
||||
// both intervals = 1s the actor retries quickly; allow 15s for slow CI.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var readRepo = new AuditLogRepository(readContext);
|
||||
var rows = await readRepo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
Assert.Single(rows);
|
||||
Assert.Equal(evt.EventId, rows[0].EventId);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
|
||||
Assert.True(stubClient.CallCount >= 2,
|
||||
$"Expected at least one failed push + one successful push; saw {stubClient.CallCount} total client calls.");
|
||||
|
||||
// The site SQLite row must have flipped to Forwarded after the
|
||||
// successful retry. ReadPendingAsync only returns Pending rows; the
|
||||
// row should NOT show up there anymore.
|
||||
var stillPending = await sqliteWriter.ReadPendingAsync(64);
|
||||
Assert.DoesNotContain(stillPending, p => p.EventId == evt.EventId);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task EndToEnd_DuplicateSubmit_OnlyOneCentralRow()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = NewSiteId();
|
||||
|
||||
await using var ingestContext = CreateContext();
|
||||
var ingestRepo = new AuditLogRepository(ingestContext);
|
||||
var ingestActor = CreateIngestActor(ingestRepo);
|
||||
|
||||
await using var sqliteWriter = CreateInMemorySqliteWriter();
|
||||
var ring = new RingBufferFallback();
|
||||
var fallback = new FallbackAuditWriter(
|
||||
sqliteWriter,
|
||||
ring,
|
||||
new NoOpAuditWriteFailureCounter(),
|
||||
NullLogger<FallbackAuditWriter>.Instance);
|
||||
|
||||
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
|
||||
CreateTelemetryActor(sqliteWriter, stubClient);
|
||||
|
||||
// Both writes carry the SAME EventId. Site SQLite's PRIMARY KEY
|
||||
// constraint and the central repo's InsertIfNotExistsAsync both
|
||||
// enforce first-write-wins, so only one central row must materialise.
|
||||
var sharedId = Guid.NewGuid();
|
||||
var evt1 = NewEvent(siteId, sharedId);
|
||||
var evt2 = NewEvent(siteId, sharedId);
|
||||
|
||||
await fallback.WriteAsync(evt1);
|
||||
await fallback.WriteAsync(evt2);
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await using var readContext = CreateContext();
|
||||
var readRepo = new AuditLogRepository(readContext);
|
||||
var rows = await readRepo.QueryAsync(
|
||||
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
|
||||
new AuditLogPaging(PageSize: 10));
|
||||
Assert.Single(rows);
|
||||
Assert.Equal(sharedId, rows[0].EventId);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user