ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/AuditWriteFailureSafetyTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.Audit;
using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using System.Net;
using System.Security.Claims;
using System.Text;
using System.Text.Encodings.Web;

namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;

/// <summary>
/// Cross-boundary safety suite verifying
/// the alog.md §13 contract: an always-throwing audit writer NEVER aborts the
/// user-facing action. Exercises every boundary that emits audit rows:
///
/// <list type="bullet">
/// <item><description>External system sync call (sync).</description></item>
/// <item><description>External system cached call (cached).</description></item>
/// <item><description>Database sync write (sync write).</description></item>
/// <item><description>Inbound API request (request).</description></item>
/// <item><description>Notification dispatcher (dispatch).</description></item>
/// </list>
///
/// <para>
/// The site-local boundaries (ESG sync/cached, DB sync) take the always-throw
/// <see cref="ThrowingAuditWriter"/> in place of the production
/// <see cref="IAuditWriter"/>; the central boundaries (Inbound API,
/// Notification dispatcher) take the always-throw
/// <see cref="ThrowingCentralAuditWriter"/> in place of
/// <see cref="ICentralAuditWriter"/>. In each case the wrapped action's
/// original return value (or original exception) must still flow back to the
/// caller untouched.
/// </para>
/// </remarks>
public class AuditWriteFailureSafetyTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
    private readonly MsSqlMigrationFixture _fixture;

    /// <summary>Initializes the suite with the shared MS SQL migration fixture.</summary>
    /// <param name="fixture">The shared migrated MS SQL database fixture.</param>
    public AuditWriteFailureSafetyTests(MsSqlMigrationFixture fixture)
    {
        _fixture = fixture;
    }

    // ---------------------------------------------------------------------
    // Always-throwing writer test doubles
    // ---------------------------------------------------------------------

    /// <summary>
    /// Site-side <see cref="IAuditWriter"/> that ALWAYS throws on
    /// <see cref="WriteAsync"/>. Used to verify that ESG / DB script-side
    /// helpers swallow the throw and return their normal result to the script.
    /// </summary>
    private sealed class ThrowingAuditWriter : IAuditWriter
    {
        private int _attempts;
        /// <summary>Gets the number of times <c>WriteAsync</c> has been attempted.</summary>
        public int Attempts => Volatile.Read(ref _attempts);

        /// <inheritdoc />
        public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
        {
            Interlocked.Increment(ref _attempts);
            return Task.FromException(new InvalidOperationException(
                "test-only ThrowingAuditWriter — audit pipeline unavailable"));
        }
    }

    /// <summary>
    /// Central-side <see cref="ICentralAuditWriter"/> that ALWAYS throws on
    /// <see cref="WriteAsync"/>. Used to verify Inbound API + Notification
    /// dispatcher absorb audit-write failures rather than propagating them
    /// into the response / state transition.
    /// </summary>
    private sealed class ThrowingCentralAuditWriter : ICentralAuditWriter
    {
        private int _attempts;
        /// <summary>Gets the number of times <c>WriteAsync</c> has been attempted.</summary>
        public int Attempts => Volatile.Read(ref _attempts);

        /// <inheritdoc />
        public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
        {
            Interlocked.Increment(ref _attempts);
            throw new InvalidOperationException(
                "test-only ThrowingCentralAuditWriter — audit subsystem unavailable");
        }
    }

    /// <summary>
    /// Site-side <see cref="ICachedCallTelemetryForwarder"/> that ALWAYS
    /// throws on <see cref="ForwardAsync"/>. The cached-call helpers absorb
    /// the throw and still return a valid <see cref="TrackedOperationId"/>.
    /// </summary>
    private sealed class ThrowingCachedForwarder : ICachedCallTelemetryForwarder
    {
        private int _attempts;
        /// <summary>Gets the number of times <c>ForwardAsync</c> has been attempted.</summary>
        public int Attempts => Volatile.Read(ref _attempts);

        /// <inheritdoc />
        public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default)
        {
            Interlocked.Increment(ref _attempts);
            return Task.FromException(new InvalidOperationException(
                "test-only ThrowingCachedForwarder — telemetry pipeline unavailable"));
        }
    }

    // ---------------------------------------------------------------------
    // Test 1 — ESG sync call still returns the original ExternalCallResult.
    // ---------------------------------------------------------------------

    /// <summary>Verifies that a synchronous external system call still returns its result when the audit writer throws.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task EsgSyncCall_BrokenAuditWriter_StillReturnsResult()
    {
        var client = Substitute.For<IExternalSystemClient>();
        var expected = new ExternalCallResult(
            Success: true,
            ResponseJson: "{\"orderId\":42}",
            ErrorMessage: null,
            WasBuffered: false);
        client.CallAsync(
                "ERP", "GetOrder",
                Arg.Any<IReadOnlyDictionary<string, object?>?>(),
                Arg.Any<CancellationToken>())
            .Returns(expected);

        var writer = new ThrowingAuditWriter();
        var helper = new ScriptRuntimeContext.ExternalSystemHelper(
            client,
            instanceName: "Plant.Pump42",
            NullLogger.Instance,
            Guid.NewGuid(),
            auditWriter: writer,
            siteId: "site-77",
            sourceScript: "ScriptActor:Sync",
            cachedForwarder: null);

        var result = await helper.Call("ERP", "GetOrder");

        Assert.Same(expected, result);
        // Proof the audit writer was attempted — otherwise the test wouldn't
        // actually exercise the safety contract.
        Assert.True(writer.Attempts >= 1,
            $"Expected audit writer to be invoked at least once; saw {writer.Attempts}.");
    }

    // ---------------------------------------------------------------------
    // Test 2 — ESG cached call still returns a TrackedOperationId.
    // ---------------------------------------------------------------------

    /// <summary>Verifies that a cached external system call still returns a tracked operation id when the audit writer throws.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task EsgCachedCall_BrokenAuditWriter_StillReturnsTrackedOperationId()
    {
        var client = Substitute.For<IExternalSystemClient>();
        // CachedCallAsync returns WasBuffered=true so the helper takes the
        // S&F-deferred path — no immediate-terminal telemetry, which keeps the
        // forwarder attempt count at exactly one (the CachedSubmit emission).
        client.CachedCallAsync(
                "ERP", "GetOrder",
                Arg.Any<IReadOnlyDictionary<string, object?>?>(),
                Arg.Any<string?>(),
                Arg.Any<CancellationToken>(),
                Arg.Any<TrackedOperationId?>())
            .Returns(new ExternalCallResult(true, null, null, WasBuffered: true));

        // BOTH the audit writer AND the cached forwarder throw — the
        // CachedSubmit emission goes through the forwarder in production, so
        // breaking only the writer wouldn't actually exercise the cached
        // path's safety contract.
        var writer = new ThrowingAuditWriter();
        var forwarder = new ThrowingCachedForwarder();
        var helper = new ScriptRuntimeContext.ExternalSystemHelper(
            client,
            instanceName: "Plant.Pump42",
            NullLogger.Instance,
            Guid.NewGuid(),
            auditWriter: writer,
            siteId: "site-77",
            sourceScript: "ScriptActor:Cached",
            cachedForwarder: forwarder);

        var trackedId = await helper.CachedCall("ERP", "GetOrder");

        // Non-default id materialised despite the forwarder failing.
        Assert.NotEqual(default, trackedId);
        Assert.True(forwarder.Attempts >= 1,
            $"Expected cached forwarder to be invoked at least once; saw {forwarder.Attempts}.");
    }

    // ---------------------------------------------------------------------
    // Test 3 — DB sync write still returns the rows-affected count.
    // ---------------------------------------------------------------------

    /// <summary>Verifies that a synchronous database write still returns the rows-affected count when the audit writer throws.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task DbSyncWrite_BrokenAuditWriter_StillReturnsRowsAffected()
    {
        const string connectionName = "machineData";
        const string instanceName = "Plant.Pump42";

        using var keepAlive = new SqliteConnection(
            "Data Source=k-safety-db;Mode=Memory;Cache=Shared");
        keepAlive.Open();

        // Schema + seed inside a unique in-memory DB.
        var dbName = $"db-{Guid.NewGuid():N}";
        var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
        using var dbKeepAlive = new SqliteConnection(connStr);
        dbKeepAlive.Open();
        using (var seed = dbKeepAlive.CreateCommand())
        {
            seed.CommandText =
                "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL);";
            seed.ExecuteNonQuery();
        }
        var inner = new SqliteConnection(connStr);
        inner.Open();

        var gateway = Substitute.For<IDatabaseGateway>();
        gateway.GetConnectionAsync(connectionName, Arg.Any<CancellationToken>())
            .Returns(inner);

        var writer = new ThrowingAuditWriter();
        var helper = new ScriptRuntimeContext.DatabaseHelper(
            gateway,
            instanceName,
            NullLogger.Instance,
            Guid.NewGuid(),
            auditWriter: writer,
            siteId: "site-77",
            sourceScript: "ScriptActor:Db",
            cachedForwarder: null);

        await using (var conn = await helper.Connection(connectionName))
        await using (var cmd = conn.CreateCommand())
        {
            cmd.CommandText = "INSERT INTO t (id, name) VALUES (1, 'safety')";
            var rows = await cmd.ExecuteNonQueryAsync();
            Assert.Equal(1, rows);
        }

        Assert.True(writer.Attempts >= 1,
            $"Expected audit writer to be invoked at least once; saw {writer.Attempts}.");
    }

    // ---------------------------------------------------------------------
    // Test 4 — Inbound API request still returns HTTP 200.
    // ---------------------------------------------------------------------

    /// <summary>Verifies that an inbound API request still returns HTTP 200 when the audit writer throws.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task InboundApi_BrokenAuditWriter_StillReturns200()
    {
        var writer = new ThrowingCentralAuditWriter();

        using var host = await BuildInboundApiHostAsync(writer, endpointStatus: 200);
        var client = host.GetTestClient();

        var resp = await client.PostAsync(
            "/api/echo",
            new StringContent("{\"x\":1}", Encoding.UTF8, "application/json"));

        Assert.Equal(HttpStatusCode.OK, resp.StatusCode);
        Assert.True(writer.Attempts >= 1,
            $"Expected central audit writer to be invoked at least once; saw {writer.Attempts}.");
    }

    // ---------------------------------------------------------------------
    // Test 5 — Notification dispatcher still transitions to Delivered.
    // ---------------------------------------------------------------------

    /// <summary>Verifies that a notification dispatch still transitions to delivered when the audit writer throws.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task NotificationDispatch_BrokenAuditWriter_StillTransitionsToDelivered()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = "test-e4-safety-" + Guid.NewGuid().ToString("N").Substring(0, 8);
        var notificationId = Guid.NewGuid();

        await SeedSmtpConfigAsync();
        await SeedNotificationAsync(notificationId, siteId);

        var adapter = new SingleOutcomeAdapter(DeliveryOutcome.Success("ops@example.com"));
        var serviceProvider = BuildNotificationDispatcherProvider(adapter);
        var throwingWriter = new ThrowingCentralAuditWriter();

        var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
            serviceProvider,
            new NotificationOutboxOptions
            {
                DispatchInterval = TimeSpan.FromHours(1),
                PurgeInterval = TimeSpan.FromDays(1),
            },
            (ICentralAuditWriter)throwingWriter,
            NullLogger<NotificationOutboxActor>.Instance)));

        actor.Tell(InternalMessages.DispatchTick.Instance);

        // Notifications table reflects the successful delivery even though
        // every audit write threw — the central direct-write writer
        // catches/logs internally and the dispatcher catches defensively too
        // (alog.md §13).
        await AwaitAssertAsync(async () =>
        {
            await using var ctx = CreateContext();
            var row = await ctx.Notifications.SingleAsync(
                n => n.NotificationId == notificationId.ToString("D"));
            Assert.Equal(NotificationStatus.Delivered, row.Status);
            Assert.NotNull(row.DeliveredAt);
        }, TimeSpan.FromSeconds(15));

        // AwaitAssert, not a bare Assert: the dispatcher persists the delivery
        // state BEFORE emitting the audit rows (DeliverOneAsync updates the
        // row, then emits Attempted, then the terminal), so seeing Delivered
        // above orders nothing with respect to the audit write — on a loaded
        // parallel run the post-write continuation can land after the poll that
        // observed Delivered, producing a spurious "saw 0". The bounded wait
        // keeps the assertion's force: the writer must actually be invoked
        // within the timeout or the test fails exactly as before.
        await AwaitAssertAsync(
            () =>
            {
                Assert.True(throwingWriter.Attempts >= 1,
                    $"Expected dispatcher to attempt audit write at least once; saw {throwingWriter.Attempts}.");
                return Task.CompletedTask;
            },
            TimeSpan.FromSeconds(15));
    }

    // ---------------------------------------------------------------------
    // Test infrastructure
    // ---------------------------------------------------------------------

    private ScadaBridgeDbContext CreateContext()
    {
        var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
            .UseSqlServer(_fixture.ConnectionString)
            .ConfigureWarnings(w => w.Ignore(
                Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
            .Options;
        return new ScadaBridgeDbContext(options);
    }

    private IServiceProvider BuildNotificationDispatcherProvider(
        INotificationDeliveryAdapter adapter)
    {
        var services = new ServiceCollection();
        services.AddDbContext<ScadaBridgeDbContext>(opts =>
            opts.UseSqlServer(_fixture.ConnectionString)
                .ConfigureWarnings(w => w.Ignore(
                    Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
        services.AddScoped<INotificationOutboxRepository>(sp =>
            new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
        services.AddScoped<INotificationRepository>(sp =>
            new NotificationRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
        services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
        return services.BuildServiceProvider();
    }

    private async Task SeedSmtpConfigAsync()
    {
        await using var ctx = CreateContext();
        // The dispatcher clamps non-positive RetryDelay to the 1-minute fallback;
        // use 1 ms so a transient outcome's NextAttemptAt is still effectively due.
        ctx.SmtpConfigurations.Add(new SmtpConfiguration(
            "smtp.example.com", "Basic", "noreply@example.com")
        {
            MaxRetries = 5,
            RetryDelay = TimeSpan.FromMilliseconds(1),
        });
        await ctx.SaveChangesAsync();
    }

    private async Task SeedNotificationAsync(Guid notificationId, string siteId)
    {
        await using var ctx = CreateContext();
        ctx.Notifications.Add(new Notification(
            notificationId.ToString("D"),
            NotificationType.Email,
            "ops-team",
            "Safety subject",
            "Safety body",
            siteId)
        {
            SourceInstanceId = "Plant.Pump42",
            SourceScript = "AlarmScript",
            CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1),
        });
        await ctx.SaveChangesAsync();
    }

    /// <summary>
    /// Single-outcome adapter — returns the same <see cref="DeliveryOutcome"/>
    /// for every call. Used by the dispatcher safety test where we only need
    /// one happy-path delivery.
    /// </summary>
    private sealed class SingleOutcomeAdapter : INotificationDeliveryAdapter
    {
        private readonly DeliveryOutcome _outcome;
        /// <summary>Initializes the adapter with the outcome to return for every delivery.</summary>
        /// <param name="outcome">The delivery outcome returned by every call.</param>
        public SingleOutcomeAdapter(DeliveryOutcome outcome) { _outcome = outcome; }
        /// <inheritdoc />
        public NotificationType Type => NotificationType.Email;
        /// <inheritdoc />
        public Task<DeliveryOutcome> DeliverAsync(
            Notification notification, CancellationToken cancellationToken = default)
            => Task.FromResult(_outcome);
    }

    /// <summary>
    /// Builds an in-memory TestHost mirroring the production inbound-API
    /// pipeline order. The supplied <paramref name="writer"/> stands in for
    /// the production <see cref="ICentralAuditWriter"/> so the safety test can
    /// install the always-throwing variant without standing up any DB.
    /// </summary>
    private static async Task<IHost> BuildInboundApiHostAsync(
        ICentralAuditWriter writer, int endpointStatus)
    {
        var hostBuilder = new HostBuilder()
            .ConfigureWebHost(webBuilder =>
            {
                webBuilder
                    .UseTestServer()
                    .ConfigureServices(services =>
                    {
                        services.AddSingleton(writer);
                        services.AddRouting();
                        services.AddAuthorization();
                        services.AddAuthentication("TestScheme")
                            .AddScheme<Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions,
                                AlwaysAuthenticatedHandler>("TestScheme", _ => { });
                    })
                    .Configure(app =>
                    {
                        app.UseRouting();
                        app.UseAuthentication();
                        app.UseAuthorization();
                        app.UseAuditWriteMiddleware();
                        app.UseEndpoints(endpoints =>
                        {
                            endpoints.MapPost("/api/{methodName}", async ctx =>
                            {
                                ctx.Items[AuditWriteMiddleware.AuditActorItemKey] = "safety-actor";
                                ctx.Response.StatusCode = endpointStatus;
                                await ctx.Response.WriteAsync("ok");
                            });
                        });
                    });
            });
        return await hostBuilder.StartAsync();
    }

    private sealed class AlwaysAuthenticatedHandler
        : Microsoft.AspNetCore.Authentication.AuthenticationHandler<
            Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions>
    {
        /// <summary>Initializes the handler with the standard authentication handler dependencies.</summary>
        /// <param name="options">The authentication scheme options monitor.</param>
        /// <param name="logger">The logger factory.</param>
        /// <param name="encoder">The URL encoder.</param>
        public AlwaysAuthenticatedHandler(
            IOptionsMonitor<Microsoft.AspNetCore.Authentication.AuthenticationSchemeOptions> options,
            Microsoft.Extensions.Logging.ILoggerFactory logger,
            UrlEncoder encoder)
            : base(options, logger, encoder) { }

        /// <inheritdoc />
        protected override Task<Microsoft.AspNetCore.Authentication.AuthenticateResult>
            HandleAuthenticateAsync()
        {
            var identity = new ClaimsIdentity(
                new[] { new Claim(ClaimTypes.Name, "framework-user") }, "TestScheme");
            var principal = new ClaimsPrincipal(identity);
            var ticket = new Microsoft.AspNetCore.Authentication.AuthenticationTicket(
                principal, "TestScheme");
            return Task.FromResult(
                Microsoft.AspNetCore.Authentication.AuthenticateResult.Success(ticket));
        }
    }
}