using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.TestHost; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Entities.Notifications; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Messages.Integration; using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; using ScadaLink.ConfigurationDatabase; using ScadaLink.ConfigurationDatabase.Repositories; using ScadaLink.ConfigurationDatabase.Tests.Migrations; using ScadaLink.InboundAPI.Middleware; using ScadaLink.NotificationOutbox; using ScadaLink.NotificationOutbox.Delivery; using ScadaLink.NotificationOutbox.Messages; using ScadaLink.SiteRuntime.Scripts; using System.Net; using System.Security.Claims; using System.Text; using System.Text.Encodings.Web; namespace ScadaLink.AuditLog.Tests.Integration; /// /// Audit Log #23 — M4 Bundle E (Task E4) cross-boundary safety suite verifying /// the alog.md §13 contract: an always-throwing audit writer NEVER aborts the /// user-facing action. Exercises every boundary that emits audit rows in M2, /// M3, and M4: /// /// /// External system sync call (M2 Bundle F). /// External system cached call (M3 Bundle E). /// Database sync write (M4 Bundle A). /// Inbound API request (M4 Bundle D). /// Notification dispatcher (M4 Bundle B). /// /// /// /// The site-local boundaries (ESG sync/cached, DB sync) take the always-throw /// in place of the production /// ; the central boundaries (Inbound API, /// Notification dispatcher) take the always-throw /// in place of /// . In each case the wrapped action's /// original return value (or original exception) must still flow back to the /// caller untouched. /// /// public class AuditWriteFailureSafetyTests : TestKit, IClassFixture { private readonly MsSqlMigrationFixture _fixture; public AuditWriteFailureSafetyTests(MsSqlMigrationFixture fixture) { _fixture = fixture; } // --------------------------------------------------------------------- // Always-throwing writer test doubles // --------------------------------------------------------------------- /// /// Site-side that ALWAYS throws on /// . Used to verify that ESG / DB script-side /// helpers swallow the throw and return their normal result to the script. /// private sealed class ThrowingAuditWriter : IAuditWriter { private int _attempts; public int Attempts => Volatile.Read(ref _attempts); public Task WriteAsync(AuditEvent evt, CancellationToken ct = default) { Interlocked.Increment(ref _attempts); return Task.FromException(new InvalidOperationException( "test-only ThrowingAuditWriter — audit pipeline unavailable")); } } /// /// Central-side that ALWAYS throws on /// . Used to verify Inbound API + Notification /// dispatcher absorb audit-write failures rather than propagating them /// into the response / state transition. /// private sealed class ThrowingCentralAuditWriter : ICentralAuditWriter { private int _attempts; public int Attempts => Volatile.Read(ref _attempts); public Task WriteAsync(AuditEvent evt, CancellationToken ct = default) { Interlocked.Increment(ref _attempts); throw new InvalidOperationException( "test-only ThrowingCentralAuditWriter — audit subsystem unavailable"); } } /// /// Site-side that ALWAYS /// throws on . The cached-call helpers absorb /// the throw and still return a valid . /// private sealed class ThrowingCachedForwarder : ICachedCallTelemetryForwarder { private int _attempts; public int Attempts => Volatile.Read(ref _attempts); public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default) { Interlocked.Increment(ref _attempts); return Task.FromException(new InvalidOperationException( "test-only ThrowingCachedForwarder — telemetry pipeline unavailable")); } } // --------------------------------------------------------------------- // Test 1 — ESG sync call still returns the original ExternalCallResult. // --------------------------------------------------------------------- [Fact] public async Task EsgSyncCall_BrokenAuditWriter_StillReturnsResult() { var client = Substitute.For(); var expected = new ExternalCallResult( Success: true, ResponseJson: "{\"orderId\":42}", ErrorMessage: null, WasBuffered: false); client.CallAsync( "ERP", "GetOrder", Arg.Any?>(), Arg.Any()) .Returns(expected); var writer = new ThrowingAuditWriter(); var helper = new ScriptRuntimeContext.ExternalSystemHelper( client, instanceName: "Plant.Pump42", NullLogger.Instance, Guid.NewGuid(), auditWriter: writer, siteId: "site-77", sourceScript: "ScriptActor:Sync", cachedForwarder: null); var result = await helper.Call("ERP", "GetOrder"); Assert.Same(expected, result); // Proof the audit writer was attempted — otherwise the test wouldn't // actually exercise the safety contract. Assert.True(writer.Attempts >= 1, $"Expected audit writer to be invoked at least once; saw {writer.Attempts}."); } // --------------------------------------------------------------------- // Test 2 — ESG cached call still returns a TrackedOperationId. // --------------------------------------------------------------------- [Fact] public async Task EsgCachedCall_BrokenAuditWriter_StillReturnsTrackedOperationId() { var client = Substitute.For(); // CachedCallAsync returns WasBuffered=true so the helper takes the // S&F-deferred path — no immediate-terminal telemetry, which keeps the // forwarder attempt count at exactly one (the CachedSubmit emission). client.CachedCallAsync( "ERP", "GetOrder", Arg.Any?>(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ExternalCallResult(true, null, null, WasBuffered: true)); // BOTH the audit writer AND the cached forwarder throw — the // CachedSubmit emission goes through the forwarder in production, so // breaking only the writer wouldn't actually exercise the cached // path's safety contract. var writer = new ThrowingAuditWriter(); var forwarder = new ThrowingCachedForwarder(); var helper = new ScriptRuntimeContext.ExternalSystemHelper( client, instanceName: "Plant.Pump42", NullLogger.Instance, Guid.NewGuid(), auditWriter: writer, siteId: "site-77", sourceScript: "ScriptActor:Cached", cachedForwarder: forwarder); var trackedId = await helper.CachedCall("ERP", "GetOrder"); // Non-default id materialised despite the forwarder failing. Assert.NotEqual(default, trackedId); Assert.True(forwarder.Attempts >= 1, $"Expected cached forwarder to be invoked at least once; saw {forwarder.Attempts}."); } // --------------------------------------------------------------------- // Test 3 — DB sync write still returns the rows-affected count. // --------------------------------------------------------------------- [Fact] public async Task DbSyncWrite_BrokenAuditWriter_StillReturnsRowsAffected() { const string connectionName = "machineData"; const string instanceName = "Plant.Pump42"; using var keepAlive = new SqliteConnection( "Data Source=k-safety-db;Mode=Memory;Cache=Shared"); keepAlive.Open(); // Schema + seed inside a unique in-memory DB. var dbName = $"db-{Guid.NewGuid():N}"; var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared"; using var dbKeepAlive = new SqliteConnection(connStr); dbKeepAlive.Open(); using (var seed = dbKeepAlive.CreateCommand()) { seed.CommandText = "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);"; seed.ExecuteNonQuery(); } var inner = new SqliteConnection(connStr); inner.Open(); var gateway = Substitute.For(); gateway.GetConnectionAsync(connectionName, Arg.Any()) .Returns(inner); var writer = new ThrowingAuditWriter(); var helper = new ScriptRuntimeContext.DatabaseHelper( gateway, instanceName, NullLogger.Instance, Guid.NewGuid(), auditWriter: writer, siteId: "site-77", sourceScript: "ScriptActor:Db", cachedForwarder: null); await using (var conn = await helper.Connection(connectionName)) await using (var cmd = conn.CreateCommand()) { cmd.CommandText = "INSERT INTO t (id, name) VALUES (1, 'safety')"; var rows = await cmd.ExecuteNonQueryAsync(); Assert.Equal(1, rows); } Assert.True(writer.Attempts >= 1, $"Expected audit writer to be invoked at least once; saw {writer.Attempts}."); } // --------------------------------------------------------------------- // Test 4 — Inbound API request still returns HTTP 200. // --------------------------------------------------------------------- [Fact] public async Task InboundApi_BrokenAuditWriter_StillReturns200() { var writer = new ThrowingCentralAuditWriter(); using var host = await BuildInboundApiHostAsync(writer, endpointStatus: 200); var client = host.GetTestClient(); var resp = await client.PostAsync( "/api/echo", new StringContent("{\"x\":1}", Encoding.UTF8, "application/json")); Assert.Equal(HttpStatusCode.OK, resp.StatusCode); Assert.True(writer.Attempts >= 1, $"Expected central audit writer to be invoked at least once; saw {writer.Attempts}."); } // --------------------------------------------------------------------- // Test 5 — Notification dispatcher still transitions to Delivered. // --------------------------------------------------------------------- [SkippableFact] public async Task NotificationDispatch_BrokenAuditWriter_StillTransitionsToDelivered() { Skip.IfNot(_fixture.Available, _fixture.SkipReason); var siteId = "test-e4-safety-" + Guid.NewGuid().ToString("N").Substring(0, 8); var notificationId = Guid.NewGuid(); await SeedSmtpConfigAsync(); await SeedNotificationAsync(notificationId, siteId); var adapter = new SingleOutcomeAdapter(DeliveryOutcome.Success("ops@example.com")); var serviceProvider = BuildNotificationDispatcherProvider(adapter); var throwingWriter = new ThrowingCentralAuditWriter(); var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor( serviceProvider, new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), PurgeInterval = TimeSpan.FromDays(1), }, (ICentralAuditWriter)throwingWriter, NullLogger.Instance))); actor.Tell(InternalMessages.DispatchTick.Instance); // Notifications table reflects the successful delivery even though // every audit write threw — the central direct-write writer // catches/logs internally and the dispatcher catches defensively too // (alog.md §13). await AwaitAssertAsync(async () => { await using var ctx = CreateContext(); var row = await ctx.Notifications.SingleAsync( n => n.NotificationId == notificationId.ToString("D")); Assert.Equal(NotificationStatus.Delivered, row.Status); Assert.NotNull(row.DeliveredAt); }, TimeSpan.FromSeconds(15)); Assert.True(throwingWriter.Attempts >= 1, $"Expected dispatcher to attempt audit write at least once; saw {throwingWriter.Attempts}."); } // --------------------------------------------------------------------- // Test infrastructure // --------------------------------------------------------------------- private ScadaLinkDbContext CreateContext() { var options = new DbContextOptionsBuilder() .UseSqlServer(_fixture.ConnectionString) .ConfigureWarnings(w => w.Ignore( Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)) .Options; return new ScadaLinkDbContext(options); } private IServiceProvider BuildNotificationDispatcherProvider( INotificationDeliveryAdapter adapter) { var services = new ServiceCollection(); services.AddDbContext(opts => opts.UseSqlServer(_fixture.ConnectionString) .ConfigureWarnings(w => w.Ignore( Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))); services.AddScoped(sp => new NotificationOutboxRepository(sp.GetRequiredService())); services.AddScoped(sp => new NotificationRepository(sp.GetRequiredService())); services.AddScoped(_ => adapter); return services.BuildServiceProvider(); } private async Task SeedSmtpConfigAsync() { await using var ctx = CreateContext(); ctx.SmtpConfigurations.Add(new SmtpConfiguration( "smtp.example.com", "Basic", "noreply@example.com") { MaxRetries = 5, RetryDelay = TimeSpan.Zero, }); await ctx.SaveChangesAsync(); } private async Task SeedNotificationAsync(Guid notificationId, string siteId) { await using var ctx = CreateContext(); ctx.Notifications.Add(new Notification( notificationId.ToString("D"), NotificationType.Email, "ops-team", "Safety subject", "Safety body", siteId) { SourceInstanceId = "Plant.Pump42", SourceScript = "AlarmScript", CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1), }); await ctx.SaveChangesAsync(); } /// /// Single-outcome adapter — returns the same /// for every call. Used by the dispatcher safety test where we only need /// one happy-path delivery. /// private sealed class SingleOutcomeAdapter : INotificationDeliveryAdapter { private readonly DeliveryOutcome _outcome; public SingleOutcomeAdapter(DeliveryOutcome outcome) { _outcome = outcome; } public NotificationType Type => NotificationType.Email; public Task DeliverAsync( Notification notification, CancellationToken cancellationToken = default) => Task.FromResult(_outcome); } /// /// Builds an in-memory TestHost mirroring the production inbound-API /// pipeline order. The supplied stands in for /// the production so the safety test can /// install the always-throwing variant without standing up any DB. /// private static async Task BuildInboundApiHostAsync( ICentralAuditWriter writer, int endpointStatus) { var hostBuilder = new HostBuilder() .ConfigureWebHost(webBuilder => { webBuilder .UseTestServer() .ConfigureServices(services => { services.AddSingleton(writer); services.AddRouting(); services.AddAuthorization(); services.AddAuthentication("TestScheme") .AddScheme("TestScheme", _ => { }); }) .Configure(app => { app.UseRouting(); app.UseAuthentication(); app.UseAuthorization(); app.UseAuditWriteMiddleware(); app.UseEndpoints(endpoints => { endpoints.MapPost("/api/{methodName}", async ctx => { ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "safety-actor"; ctx.Response.StatusCode = endpointStatus; await ctx.Response.WriteAsync("ok"); }); }); }); }); return await hostBuilder.StartAsync(); } private sealed class AlwaysAuthenticatedHandler : Microsoft.AspNetCore.Authentication.AuthenticationHandler< Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions> { public AlwaysAuthenticatedHandler( IOptionsMonitor options, Microsoft.Extensions.Logging.ILoggerFactory logger, UrlEncoder encoder) : base(options, logger, encoder) { } protected override Task HandleAuthenticateAsync() { var identity = new ClaimsIdentity( new[] { new Claim(ClaimTypes.Name, "framework-user") }, "TestScheme"); var principal = new ClaimsPrincipal(identity); var ticket = new Microsoft.AspNetCore.Authentication.AuthenticationTicket( principal, "TestScheme"); return Task.FromResult( Microsoft.AspNetCore.Authentication.AuthenticateResult.Success(ticket)); } } }