ScadaBridge/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs

using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities;

namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;

/// <summary>
/// EF Core implementation of <see cref="IAuditLogRepository"/>. See the interface
/// for the append-only contract; this class only adds notes on the data-access
/// strategy used by each method.
/// </summary>
public class AuditLogRepository : IAuditLogRepository
{
    // SQL Server error numbers for duplicate-key violations on
    // UX_AuditLog_EventId. 2601 is a unique-index violation; 2627 is a
    // primary-key/unique-constraint violation. The IF NOT EXISTS … INSERT
    // pattern has a check-then-act race window — two sessions can both pass
    // the EXISTS check and then both attempt the INSERT — and the loser
    // surfaces as one of these errors. Idempotency demands we swallow them.
    private const int SqlErrorUniqueIndexViolation = 2601;
    private const int SqlErrorPrimaryKeyViolation = 2627;

    private readonly ScadaBridgeDbContext _context;
    private readonly ILogger<AuditLogRepository> _logger;

    /// <summary>Initializes a new instance of the AuditLogRepository class.</summary>
    /// <param name="context">The database context.</param>
    /// <param name="logger">Optional logger instance.</param>
    public AuditLogRepository(ScadaBridgeDbContext context, ILogger<AuditLogRepository>? logger = null)
    {
        _context = context ?? throw new ArgumentNullException(nameof(context));
        _logger = logger ?? NullLogger<AuditLogRepository>.Instance;
    }

    /// <inheritdoc />
    public async Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
    {
        if (evt is null)
        {
            throw new ArgumentNullException(nameof(evt));
        }

        // Write the 10 canonical columns DIRECTLY — no Decompose.
        // The five queryability columns (Kind/Status/SourceSiteId/ExecutionId/
        // ParentExecutionId) plus IngestedAtUtc are PERSISTED computed columns on
        // dbo.AuditLog; SQL Server derives them from DetailsJson at INSERT, so they
        // are intentionally absent from this column list (writing a computed column
        // is an error). The canonical OccurredAtUtc is UTC by construction; store a
        // Kind=Utc DateTime so downstream UTC/local conversions are safe.
        var occurred = DateTime.SpecifyKind(evt.OccurredAtUtc.UtcDateTime, DateTimeKind.Utc);

        // Canonical Actor is a required non-null string; an empty Actor maps to a
        // NULL column (legacy/central rows stored null for system/anon).
        string? actor = string.IsNullOrEmpty(evt.Actor) ? null : evt.Actor;

        // Outcome / Category are varchar columns (Outcome via HasConversion<string>;
        // Category carries the channel name). Bind as strings rather than relying on
        // parameter type inference.
        var outcome = evt.Outcome.ToString();
        string? category = evt.Category;

        // FormattableString interpolation parameterises every value (no concatenation),
        // so this is safe against injection even for the string columns.
        try
        {
            await _context.Database.ExecuteSqlInterpolatedAsync(
                $@"IF NOT EXISTS (SELECT 1 FROM dbo.AuditLog WHERE EventId = {evt.EventId})
INSERT INTO dbo.AuditLog
    (EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson)
VALUES
    ({evt.EventId}, {occurred}, {actor}, {evt.Action}, {outcome}, {category}, {evt.Target}, {evt.SourceNode}, {evt.CorrelationId}, {evt.DetailsJson});",
                ct);
        }
        catch (SqlException ex) when (
            ex.Number == SqlErrorUniqueIndexViolation
            || ex.Number == SqlErrorPrimaryKeyViolation)
        {
            // Two concurrent sessions both passed the IF NOT EXISTS check and
            // both attempted the INSERT — the loser raises 2601/2627 against
            // UX_AuditLog_EventId. First-write-wins idempotency is already the
            // documented contract for this method, so the race outcome is
            // semantically a no-op. Swallow at Debug; other SqlExceptions
            // bubble.
            _logger.LogDebug(
                ex,
                "InsertIfNotExistsAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for EventId {EventId}; treating as no-op.",
                ex.Number,
                evt.EventId);
        }
    }

    /// <inheritdoc />
    public async Task<IReadOnlyList<AuditEvent>> QueryAsync(
        AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
    {
        if (filter is null)
        {
            throw new ArgumentNullException(nameof(filter));
        }

        if (paging is null)
        {
            throw new ArgumentNullException(nameof(paging));
        }

        // The filter predicates bind to the canonical columns and the
        // persisted computed columns directly — Channel→Category, Kind/Status/
        // SourceSiteId/ExecutionId/ParentExecutionId are computed columns. The
        // materialized rows are projected to the canonical record by reading the 10
        // canonical columns (no 24-column Recompose).
        var query = _context.Set<AuditLogRow>().AsNoTracking();

        // Multi-value dimensions: a null OR empty list means "no constraint"
        // (the { Count: > 0 } guard prevents an empty list collapsing to a
        // WHERE 1=0). A non-empty list translates to a SQL IN (…) via EF Core's
        // IReadOnlyList<T>.Contains support — server-side, no client-eval.
        if (filter.Channels is { Count: > 0 } channels)
        {
            query = query.Where(e => channels.Contains(e.Channel));
        }

        if (filter.Kinds is { Count: > 0 } kinds)
        {
            query = query.Where(e => kinds.Contains(e.Kind));
        }

        if (filter.Statuses is { Count: > 0 } statuses)
        {
            query = query.Where(e => statuses.Contains(e.Status));
        }

        if (filter.SourceSiteIds is { Count: > 0 } sourceSiteIds)
        {
            query = query.Where(e => e.SourceSiteId != null && sourceSiteIds.Contains(e.SourceSiteId));
        }

        // SourceNode filter mirrors SourceSiteIds: a non-empty list translates to
        // SQL IN (…); NULL SourceNode rows are excluded when the filter is set.
        if (filter.SourceNodes is { Count: > 0 } sourceNodes)
        {
            query = query.Where(e => e.SourceNode != null && sourceNodes.Contains(e.SourceNode));
        }

        if (!string.IsNullOrEmpty(filter.Target))
        {
            var target = filter.Target;
            query = query.Where(e => e.Target == target);
        }

        if (!string.IsNullOrEmpty(filter.Actor))
        {
            var actor = filter.Actor;
            query = query.Where(e => e.Actor == actor);
        }

        if (filter.CorrelationId is { } correlationId)
        {
            query = query.Where(e => e.CorrelationId == correlationId);
        }

        if (filter.ExecutionId is { } executionId)
        {
            query = query.Where(e => e.ExecutionId == executionId);
        }

        if (filter.ParentExecutionId is { } parentExecutionId)
        {
            query = query.Where(e => e.ParentExecutionId == parentExecutionId);
        }

        if (filter.FromUtc is { } fromUtc)
        {
            query = query.Where(e => e.OccurredAtUtc >= fromUtc);
        }

        if (filter.ToUtc is { } toUtc)
        {
            query = query.Where(e => e.OccurredAtUtc <= toUtc);
        }

        // Keyset cursor on (OccurredAtUtc desc, EventId desc).
        if (paging.AfterOccurredAtUtc is { } afterOccurred && paging.AfterEventId is { } afterEventId)
        {
            query = query.Where(e =>
                e.OccurredAtUtc < afterOccurred
                || (e.OccurredAtUtc == afterOccurred && e.EventId.CompareTo(afterEventId) < 0));
        }

        var rows = await query
            .OrderByDescending(e => e.OccurredAtUtc)
            .ThenByDescending(e => e.EventId)
            .Take(paging.PageSize)
            .ToListAsync(ct);

        return rows.Select(RowToCanonical).ToList();
    }

    /// <summary>
    /// Build the canonical <see cref="AuditEvent"/> DIRECTLY from the
    /// 10 canonical columns of a materialized <see cref="AuditLogRow"/> read back from
    /// <c>dbo.AuditLog</c> — no 24-column <c>Recompose</c>, because the table now holds
    /// the canonical shape (every ScadaBridge domain field already lives in
    /// <c>DetailsJson</c>). The persisted computed columns are read helpers only and
    /// are not part of the canonical record. <see cref="AuditLogRow.Channel"/> is the
    /// canonical <c>Category</c> column (Category = channel name for ScadaBridge).
    /// </summary>
    private static AuditEvent RowToCanonical(AuditLogRow row)
        => new()
        {
            EventId = row.EventId,
            OccurredAtUtc = new DateTimeOffset(
                DateTime.SpecifyKind(row.OccurredAtUtc, DateTimeKind.Utc)),
            Actor = row.Actor ?? string.Empty,
            Action = row.Action,
            Outcome = row.Outcome,
            Category = row.Channel.ToString(),
            Target = row.Target,
            SourceNode = row.SourceNode,
            CorrelationId = row.CorrelationId,
            DetailsJson = row.DetailsJson,
        };

    /// <inheritdoc />
    public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
    {
        // The drop-and-rebuild batch below runs via
        // ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side
        // BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying
        // execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on
        // a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS
        // guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
        //
        // Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
        // provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
        // arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
        var commandTimeoutSeconds = commandTimeout is { } ct2 ? (int)ct2.TotalSeconds : (int?)null;

        // GUID-suffixed staging name: prevents collision with any concurrent
        // purge attempt and avoids polluting the AuditLog object namespace with
        // a predictable identifier.
        var stagingTableName = $"AuditLog_Staging_{Guid.NewGuid():N}";

        // ISO 8601 in UTC — SQL Server's datetime2 literal parser accepts this
        // unambiguously and the value is round-trip-safe across SET DATEFORMAT
        // settings. Use datetime2(7) precision (.fffffff) so a future
        // non-midnight or sub-second boundary doesn't silently round to the
        // wrong partition (today the migration only seeds at T00:00:00 exactly,
        // but the format string is on the boundary value's own contract — match
        // it to the column precision rather than to the current seed pattern).
        var monthBoundaryStr = monthBoundary.ToUniversalTime().ToString("yyyy-MM-dd HH:mm:ss.fffffff");

        // Two-statement batch: the first SELECT samples the per-partition row
        // count BEFORE the dance so we can report it back to the purge actor;
        // the second batch performs the drop-and-rebuild. We use OUTPUT-style
        // variables wired through @@ROWCOUNT after the SWITCH is not viable
        // because SWITCH is a metadata-only operation that doesn't move rows in
        // a way @@ROWCOUNT can observe.
        var sampleSql = $@"
            SELECT COUNT_BIG(*) FROM dbo.AuditLog
            WHERE $PARTITION.pf_AuditLog_Month(OccurredAtUtc) =
                  $partition.pf_AuditLog_Month('{monthBoundaryStr}');";

        var sql = $@"
        BEGIN TRY
            BEGIN TRANSACTION;

            -- 1. Drop the non-aligned unique index. ALTER TABLE SWITCH refuses
            --    to run while it exists.
            IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
                DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;

            -- 2. Staging table on [PRIMARY] (non-partitioned) with column shapes
            --    byte-identical to the dbo.AuditLog — INCLUDING the persisted
            --    computed columns, whose definitions must match EXACTLY (same
            --    expression text + PERSISTED) or ALTER TABLE ... SWITCH PARTITION
            --    rejects the operation with msg 4904/4948. The ordinal order also
            --    matches dbo.AuditLog_v2 (the CollapseAuditLogToCanonical migration):
            --    10 canonical columns first, then the 6 computed columns.
            CREATE TABLE dbo.[{stagingTableName}] (
                EventId            uniqueidentifier NOT NULL,
                OccurredAtUtc      datetime2(7)     NOT NULL,
                Actor              nvarchar(256)    NULL,
                Action             varchar(64)      NOT NULL,
                Outcome            varchar(16)      NOT NULL,
                Category           varchar(32)      NOT NULL,
                Target             nvarchar(256)    NULL,
                SourceNode         varchar(64)      NULL,
                CorrelationId      uniqueidentifier NULL,
                DetailsJson        nvarchar(max)    NULL,
                Kind               AS JSON_VALUE(DetailsJson,'$.kind') PERSISTED,
                Status             AS JSON_VALUE(DetailsJson,'$.status') PERSISTED,
                SourceSiteId       AS JSON_VALUE(DetailsJson,'$.sourceSiteId') PERSISTED,
                ExecutionId        AS CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier) PERSISTED,
                ParentExecutionId  AS CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier) PERSISTED,
                IngestedAtUtc      AS CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7)),
                CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc)
            ) ON [PRIMARY];

            -- 3. Switch the partition out. $partition.pf_AuditLog_Month returns
            --    the partition number that contains the supplied boundary value;
            --    SWITCH PARTITION N moves that partition's pages to the staging
            --    table (metadata-only, no row copying).
            DECLARE @partitionNumber int = $partition.pf_AuditLog_Month('{monthBoundaryStr}');
            DECLARE @sql nvarchar(max) = 'ALTER TABLE dbo.AuditLog SWITCH PARTITION ' + CAST(@partitionNumber AS nvarchar(10)) + ' TO dbo.[{stagingTableName}];';
            EXEC sp_executesql @sql;

            -- 4. Drop staging — the rows are discarded here. This is the purge.
            DROP TABLE dbo.[{stagingTableName}];

            -- 5. Rebuild the non-aligned unique index. Live traffic that hit the
            --    table during steps 1-4 saw composite-PK uniqueness only; from
            --    here on, single-column EventId uniqueness is restored.
            CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];

            COMMIT TRANSACTION;
        END TRY
        BEGIN CATCH
            IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION;

            -- Best-effort staging cleanup. The DROP INDEX in step 1 is now
            -- rolled back (so the index is back), but the staging table from
            -- step 2 may or may not survive the rollback depending on the
            -- failure point. Guard the DROP so a missing staging table doesn't
            -- mask the original error.
            IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}];

            -- Idempotent index rebuild — covers the niche case where ROLLBACK
            -- failed to restore UX_AuditLog_EventId (or the failure happened
            -- AFTER the COMMIT, which shouldn't be possible inside this TRY
            -- but is cheap insurance). Without this, a failed switch could
            -- leave the live table without its idempotency-supporting index.
            IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
                CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];

            -- Surface the original error to the caller — the purge actor logs
            -- and continues with the next boundary.
            THROW;
        END CATCH;";

        // Sample the row count before the switch. The sample is best-effort
        // (no transaction wrapping the sample-then-switch pair) because the
        // central singleton is the only writer to this RPC and a daily-purge
        // tick doesn't compete with concurrent SwitchOut callers. A
        // concurrent INSERT racing the sample under-reports by at most a
        // few rows, which is acceptable for an "approximate" purged-row
        // count surfaced via AuditLogPurgedEvent.
        long rowsDeleted = 0;
        var conn = _context.Database.GetDbConnection();
        var openedHere = false;
        if (conn.State != System.Data.ConnectionState.Open)
        {
            await conn.OpenAsync(ct).ConfigureAwait(false);
            openedHere = true;
        }
        try
        {
            await using (var sampleCmd = conn.CreateCommand())
            {
                sampleCmd.CommandText = sampleSql;
                if (commandTimeoutSeconds is { } sampleTimeout)
                {
                    sampleCmd.CommandTimeout = sampleTimeout;
                }
                var sampleResult = await sampleCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
                if (sampleResult is not null && sampleResult is not DBNull)
                {
                    rowsDeleted = Convert.ToInt64(sampleResult);
                }
            }
        }
        finally
        {
            if (openedHere)
            {
                await conn.CloseAsync().ConfigureAwait(false);
            }
        }

        // Apply the maintenance timeout to the drop-and-rebuild dance. The context is scoped per
        // purge tick (see AuditLogPurgeActor.OnTickAsync's CreateAsyncScope), so a restore is not
        // strictly required — but we restore the previous value in a finally for hygiene in case a
        // future caller reuses the context for other work after the switch-out.
        var previousTimeout = _context.Database.GetCommandTimeout();
        if (commandTimeout is { } maintenanceTimeout)
        {
            _context.Database.SetCommandTimeout(maintenanceTimeout);
        }
        try
        {
            await _context.Database.ExecuteSqlRawAsync(sql, ct);
        }
        finally
        {
            if (commandTimeout is not null)
            {
                _context.Database.SetCommandTimeout(previousTimeout);
            }
        }

        return rowsDeleted;
    }

    /// <inheritdoc />
    public async Task<long> PurgeChannelOlderThanAsync(
        string channel,
        DateTime threshold,
        int batchSize,
        TimeSpan? commandTimeout = null,
        CancellationToken ct = default)
    {
        if (string.IsNullOrWhiteSpace(channel))
        {
            throw new ArgumentException("Channel must be a non-empty channel name.", nameof(channel));
        }

        if (batchSize <= 0)
        {
            throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be > 0.");
        }

        var thresholdUtc = DateTime.SpecifyKind(threshold.ToUniversalTime(), DateTimeKind.Utc);

        // Maintenance timeout in whole seconds; null leaves the provider default (arch-review 04 S2).
        var commandTimeoutSeconds = commandTimeout is { } cto ? (int)cto.TotalSeconds : (int?)null;

        // Per-channel retention override purge. This is the ONLY DELETE
        // against dbo.AuditLog in the codebase and it runs on the purge/maintenance
        // path, NOT the append-only writer role (which has INSERT + SELECT only — see
        // the DENY UPDATE/DENY DELETE grants in CollapseAuditLogToCanonical). The
        // AuditLog append-only CI guard (AuditLogAppendOnlyGuardTests) is intentionally
        // widened to allow ONLY the single marked DELETE below; any other UPDATE/DELETE
        // targeting AuditLog still trips the guard.
        //
        // Bounded + idempotent: DELETE TOP (@batch) caps the log/lock footprint per
        // statement; the loop repeats until a batch deletes zero rows, so re-running
        // after a crash mid-loop simply resumes. Category is the canonical
        // channel-name column (e.g. 'ApiOutbound'); Action holds "{channel}.{kind}" so
        // it is NOT the right column to match a bare channel name against.
        //
        // The trailing AUDIT-PURGE-ALLOWED marker on the DELETE line below is the
        // single narrow exemption the append-only CI guard (AuditLogAppendOnlyGuardTests)
        // recognizes; any other UPDATE/DELETE targeting AuditLog still trips the guard.
        const string deleteBatchSql =
            "DELETE TOP (@batch) FROM dbo.AuditLog WHERE Category = @channel AND OccurredAtUtc < @threshold;"; // AUDIT-PURGE-ALLOWED: per-channel retention override, maintenance path

        long totalDeleted = 0;

        var conn = _context.Database.GetDbConnection();
        var openedHere = false;
        if (conn.State != System.Data.ConnectionState.Open)
        {
            await conn.OpenAsync(ct).ConfigureAwait(false);
            openedHere = true;
        }

        try
        {
            while (true)
            {
                ct.ThrowIfCancellationRequested();

                await using var cmd = conn.CreateCommand();
                cmd.CommandText = deleteBatchSql;
                if (commandTimeoutSeconds is { } batchTimeout)
                {
                    cmd.CommandTimeout = batchTimeout;
                }

                var pBatch = cmd.CreateParameter();
                pBatch.ParameterName = "@batch";
                pBatch.Value = batchSize;
                cmd.Parameters.Add(pBatch);

                var pChannel = cmd.CreateParameter();
                pChannel.ParameterName = "@channel";
                pChannel.Value = channel;
                cmd.Parameters.Add(pChannel);

                var pThreshold = cmd.CreateParameter();
                pThreshold.ParameterName = "@threshold";
                pThreshold.Value = thresholdUtc;
                cmd.Parameters.Add(pThreshold);

                var rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
                if (rows <= 0)
                {
                    break;
                }

                totalDeleted += rows;
            }
        }
        finally
        {
            if (openedHere)
            {
                await conn.CloseAsync().ConfigureAwait(false);
            }
        }

        return totalDeleted;
    }

    /// <inheritdoc />
    public async Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
        DateTime threshold,
        CancellationToken ct = default)
    {
        var thresholdUtc = threshold.ToUniversalTime();
        var thresholdStr = thresholdUtc.ToString("yyyy-MM-dd HH:mm:ss.fffffff");

        // Per-partition MAX over the live table. We materialise the boundary
        // list first (24 rows) then LEFT JOIN to the MAX aggregate so empty
        // partitions surface as NULL and get filtered out by the WHERE clause.
        var sql = $@"
            WITH Boundaries AS (
                SELECT CAST(rv.value AS datetime2(7)) AS BoundaryValue,
                       rv.boundary_id AS BoundaryId
                FROM sys.partition_range_values rv
                INNER JOIN sys.partition_functions pf ON rv.function_id = pf.function_id
                WHERE pf.name = 'pf_AuditLog_Month'
            )
            SELECT b.BoundaryValue
            FROM Boundaries b
            CROSS APPLY (
                SELECT MAX(a.OccurredAtUtc) AS MaxOccurredAt
                FROM dbo.AuditLog a
                WHERE $PARTITION.pf_AuditLog_Month(a.OccurredAtUtc) = b.BoundaryId + 1
            ) x
            WHERE x.MaxOccurredAt IS NOT NULL
              AND x.MaxOccurredAt < CAST('{thresholdStr}' AS datetime2(7))
            ORDER BY b.BoundaryValue;";

        var conn = _context.Database.GetDbConnection();
        var openedHere = false;
        if (conn.State != System.Data.ConnectionState.Open)
        {
            await conn.OpenAsync(ct).ConfigureAwait(false);
            openedHere = true;
        }

        var results = new List<DateTime>();
        try
        {
            await using var cmd = conn.CreateCommand();
            cmd.CommandText = sql;
            await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
            while (await reader.ReadAsync(ct).ConfigureAwait(false))
            {
                // SQL Server's datetime2 surfaces as DateTimeKind.Unspecified
                // through ADO.NET (the column type carries no offset/kind).
                // Boundary values are stored in UTC, so re-tag the kind here —
                // matches the explicit defence in
                // AuditLogPartitionMaintenance.GetMaxBoundaryAsync and prevents
                // downstream .ToLocalTime()/.ToUniversalTime() conversions
                // from silently treating the value as local time.
                results.Add(DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc));
            }
        }
        finally
        {
            if (openedHere)
            {
                await conn.CloseAsync().ConfigureAwait(false);
            }
        }

        return results;
    }

    /// <inheritdoc />
    public async Task<AuditLogKpiSnapshot> GetKpiSnapshotAsync(
        TimeSpan window,
        DateTime? nowUtc = null,
        CancellationToken ct = default)
    {
        var anchorUtc = (nowUtc ?? DateTime.UtcNow).ToUniversalTime();
        var thresholdUtc = anchorUtc - window;

        // ExecuteSqlInterpolated parameterises every interpolation — the enum
        // discriminators are passed as varchar parameters that match the
        // varchar(32) Status column (HasConversion<string>()).
        var failedStr = nameof(Commons.Types.Enums.AuditStatus.Failed);
        var parkedStr = nameof(Commons.Types.Enums.AuditStatus.Parked);
        var discardedStr = nameof(Commons.Types.Enums.AuditStatus.Discarded);

        long total = 0;
        long errors = 0;

        var conn = _context.Database.GetDbConnection();
        var openedHere = false;
        if (conn.State != System.Data.ConnectionState.Open)
        {
            await conn.OpenAsync(ct).ConfigureAwait(false);
            openedHere = true;
        }

        try
        {
            await using var cmd = conn.CreateCommand();
            // Named parameters keep the prepared statement cache stable across
            // calls — only the values change. COUNT_BIG returns a bigint so
            // we read into long even when the running total fits in int.
            cmd.CommandText = @"
                SELECT
                    COUNT_BIG(*) AS Total,
                    SUM(CASE WHEN Status IN (@failed, @parked, @discarded) THEN 1 ELSE 0 END) AS Errors
                FROM dbo.AuditLog
                WHERE OccurredAtUtc >= @threshold
                  AND OccurredAtUtc <= @anchor;";

            var pThreshold = cmd.CreateParameter();
            pThreshold.ParameterName = "@threshold";
            pThreshold.Value = thresholdUtc;
            cmd.Parameters.Add(pThreshold);

            var pAnchor = cmd.CreateParameter();
            pAnchor.ParameterName = "@anchor";
            pAnchor.Value = anchorUtc;
            cmd.Parameters.Add(pAnchor);

            var pFailed = cmd.CreateParameter();
            pFailed.ParameterName = "@failed";
            pFailed.Value = failedStr;
            cmd.Parameters.Add(pFailed);

            var pParked = cmd.CreateParameter();
            pParked.ParameterName = "@parked";
            pParked.Value = parkedStr;
            cmd.Parameters.Add(pParked);

            var pDiscarded = cmd.CreateParameter();
            pDiscarded.ParameterName = "@discarded";
            pDiscarded.Value = discardedStr;
            cmd.Parameters.Add(pDiscarded);

            await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
            if (await reader.ReadAsync(ct).ConfigureAwait(false))
            {
                // SUM over an empty set is NULL; COUNT_BIG over an empty set is 0.
                total = reader.IsDBNull(0) ? 0L : reader.GetInt64(0);
                errors = reader.IsDBNull(1) ? 0L : Convert.ToInt64(reader.GetValue(1));
            }
        }
        finally
        {
            if (openedHere)
            {
                await conn.CloseAsync().ConfigureAwait(false);
            }
        }

        return new AuditLogKpiSnapshot(
            TotalEventsLastHour: total,
            ErrorEventsLastHour: errors,
            BacklogTotal: 0L,
            AsOfUtc: anchorUtc);
    }

    // Hard ceiling on chain depth for both the upward walk and the downward
    // recursive CTE. The ParentExecutionId graph is a tree (acyclic by
    // construction — each execution is minted fresh, its parent always
    // pre-exists), so this is purely a guard against corrupt/pathological data:
    // a cycle must surface as a bounded error rather than hang the server.
    // Chains are shallow (1-2 levels typical) so the guard is never reached in
    // practice.
    private const int ExecutionChainMaxDepth = 32;

    // Execution trees span minutes, not years — an inbound request and every script it
    // spawns all fire inside one operational burst. Anchoring the Edges CTE scan to a
    // narrow window around the root's first event lets SQL Server eliminate every
    // AuditLog partition outside it, turning a full-table DISTINCT scan into a seek on
    // IX_AuditLog_Execution over one or two months (arch-review 04, P2). The slack absorbs
    // minor clock skew before the root; the max span is the documented traversal bound —
    // a genuine descendant stamped beyond it is excluded by design.
    private static readonly TimeSpan ExecutionTreeWindowSlack = TimeSpan.FromHours(1);
    private static readonly TimeSpan ExecutionTreeMaxSpan = TimeSpan.FromDays(7);

    /// <inheritdoc />
    public async Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
        Guid executionId,
        CancellationToken ct = default)
    {
        var conn = _context.Database.GetDbConnection();
        var openedHere = false;
        if (conn.State != System.Data.ConnectionState.Open)
        {
            await conn.OpenAsync(ct).ConfigureAwait(false);
            openedHere = true;
        }

        try
        {
            // --- Phase 1: walk up to the root ---------------------------------
            // Climb ParentExecutionId until a node has no parent (root) or the
            // parent execution has no rows of its own (purged/stub — the climb
            // cannot continue past a row-less node). The depth cap guards
            // against a cycle in corrupt data; a tree never reaches it.
            var rootExecutionId = executionId;
            for (var depth = 0; depth < ExecutionChainMaxDepth; depth++)
            {
                Guid? parent;
                await using (var upCmd = conn.CreateCommand())
                {
                    upCmd.CommandText =
                        "SELECT TOP 1 ParentExecutionId FROM dbo.AuditLog " +
                        "WHERE ExecutionId = @cur AND ParentExecutionId IS NOT NULL;";
                    var pCur = upCmd.CreateParameter();
                    pCur.ParameterName = "@cur";
                    pCur.Value = rootExecutionId;
                    upCmd.Parameters.Add(pCur);

                    var result = await upCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
                    parent = result is null or DBNull ? null : (Guid)result;
                }

                if (parent is null)
                {
                    // No parent row for the current node — it is the root (or a
                    // row-less stub at the top of what survives). Stop climbing.
                    break;
                }

                rootExecutionId = parent.Value;
            }

            // --- Between phases: bound the down-walk to a root-anchored window ---
            // The root's first event timestamp anchors a [rootFirst - slack, rootFirst + maxSpan)
            // window used to prune the Edges CTE scan to a handful of partitions. A row-less stub
            // root (purged/no-action parent) has no MIN — leave the scan unbounded in that
            // degenerate case (correctness over speed).
            DateTime? rootFirstOccurred;
            await using (var winCmd = conn.CreateCommand())
            {
                winCmd.CommandText =
                    "SELECT MIN(OccurredAtUtc) FROM dbo.AuditLog WHERE ExecutionId = @root;";
                var pWinRoot = winCmd.CreateParameter();
                pWinRoot.ParameterName = "@root";
                pWinRoot.Value = rootExecutionId;
                winCmd.Parameters.Add(pWinRoot);

                var winResult = await winCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
                rootFirstOccurred = winResult is null or DBNull ? null : (DateTime)winResult;
            }

            // Build the optional window predicate for the Edges CTE. When the root has rows the
            // scan is confined to the window (partition elimination); otherwise it stays unbounded.
            var edgesWindowPredicate = rootFirstOccurred is null
                ? string.Empty
                : "AND OccurredAtUtc >= @winStart AND OccurredAtUtc < @winEnd";

            // --- Phase 2: walk down from the root via a recursive CTE ---------
            // Edges : a non-recursive, DISTINCT (ExecutionId, ParentExecutionId)
            //         edge set distilled from AuditLog. Recursing over edges
            //         instead of raw rows means an execution with N audit rows
            //         contributes ONE recursion path, not N — MAXRECURSION
            //         bounds depth, not per-level width, so the raw-row form
            //         could fan out badly. One edge per execution because all
            //         rows of an execution share a single ParentExecutionId
            //         (see the MIN(...) note on the final projection).
            // Chain : seeded at the root edge, recursively joins each edge whose
            //         ParentExecutionId is an ExecutionId already in the chain.
            //         Each edge carries its own ParentExecutionId, so the chain
            //         of edges already surfaces every execution id in the tree
            //         — including a row-less stub parent, which appears as the
            //         ParentExecutionId of its child's edge. No separate
            //         union-back CTE is needed.
            // Final   : collect every distinct execution id reachable from the
            //         chain (each edge's ExecutionId plus its non-null
            //         ParentExecutionId), LEFT JOIN back to AuditLog and
            //         GROUP BY so a stub parent — which owns no edge of its own
            //         because it emitted no rows — still surfaces as a node with
            //         RowCount 0 and NULL aggregates.
            var nodes = new List<ExecutionTreeNode>();
            await using (var downCmd = conn.CreateCommand())
            {
                downCmd.CommandText = $@"
                    WITH Edges AS (
                        SELECT DISTINCT ExecutionId, ParentExecutionId
                        FROM dbo.AuditLog
                        WHERE ExecutionId IS NOT NULL
                        {edgesWindowPredicate}
                    ),
                    Chain AS (
                        -- Anchor: the root execution id, seeded as a literal so
                        -- it is present even when the root is a row-less stub
                        -- (a purged/no-action parent owns no edge of its own).
                        -- The root is parentless by construction — the upward
                        -- walk stopped there — so its ParentExecutionId is NULL.
                        SELECT CAST(@root AS uniqueidentifier) AS ExecutionId,
                               CAST(NULL AS uniqueidentifier) AS ParentExecutionId
                        UNION ALL
                        SELECT e.ExecutionId, e.ParentExecutionId
                        FROM Edges e
                        INNER JOIN Chain c ON e.ParentExecutionId = c.ExecutionId
                    ),
                    ChainIds AS (
                        SELECT ExecutionId FROM Chain
                        UNION
                        SELECT ParentExecutionId FROM Chain
                        WHERE ParentExecutionId IS NOT NULL
                    )
                    -- ExecutionId / ParentExecutionId / SourceSiteId
                    -- are persisted computed columns (same names); Channel is now the
                    -- canonical Category column (Category = channel name, so the
                    -- Channels aggregate still yields channel names); SourceInstanceId
                    -- is no longer a column — read it from DetailsJson via JSON_VALUE.
                    -- ParentExecutionId / SourceSiteId / SourceInstanceId are derived
                    -- via MIN: every audit row of one execution carries the SAME value
                    -- (stamped once per script run / inbound request) — MIN simply
                    -- picks that one shared value, not collapsing a genuine
                    -- disagreement across rows.
                    SELECT
                        ids.ExecutionId                                AS [ExecutionId],
                        MIN(a.ParentExecutionId)                       AS [ParentExecutionId],
                        COUNT(a.EventId)                               AS [RowCount],
                        (SELECT STRING_AGG(d.Category, ',')
                         FROM (SELECT DISTINCT a2.Category FROM dbo.AuditLog a2
                               WHERE a2.ExecutionId = ids.ExecutionId) d) AS [Channels],
                        (SELECT STRING_AGG(d.Status, ',')
                         FROM (SELECT DISTINCT a2.Status FROM dbo.AuditLog a2
                               WHERE a2.ExecutionId = ids.ExecutionId) d) AS [Statuses],
                        MIN(a.SourceSiteId)                            AS [SourceSiteId],
                        MIN(JSON_VALUE(a.DetailsJson,'$.sourceInstanceId')) AS [SourceInstanceId],
                        MIN(a.OccurredAtUtc)                           AS [FirstOccurredAtUtc],
                        MAX(a.OccurredAtUtc)                           AS [LastOccurredAtUtc]
                    FROM ChainIds ids
                    LEFT JOIN dbo.AuditLog a ON a.ExecutionId = ids.ExecutionId
                    GROUP BY ids.ExecutionId
                    OPTION (MAXRECURSION {ExecutionChainMaxDepth});";

                var pRoot = downCmd.CreateParameter();
                pRoot.ParameterName = "@root";
                pRoot.Value = rootExecutionId;
                downCmd.Parameters.Add(pRoot);

                if (rootFirstOccurred is { } rootFirst)
                {
                    var pWinStart = downCmd.CreateParameter();
                    pWinStart.ParameterName = "@winStart";
                    pWinStart.Value = rootFirst - ExecutionTreeWindowSlack;
                    downCmd.Parameters.Add(pWinStart);

                    var pWinEnd = downCmd.CreateParameter();
                    pWinEnd.ParameterName = "@winEnd";
                    pWinEnd.Value = rootFirst + ExecutionTreeMaxSpan;
                    downCmd.Parameters.Add(pWinEnd);
                }

                await using var reader = await downCmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
                while (await reader.ReadAsync(ct).ConfigureAwait(false))
                {
                    var nodeExecutionId = reader.GetGuid(0);
                    Guid? parentExecutionId = reader.IsDBNull(1) ? null : reader.GetGuid(1);
                    var rowCount = reader.GetInt32(2);
                    var channels = SplitAggregate(reader.IsDBNull(3) ? null : reader.GetString(3));
                    var statuses = SplitAggregate(reader.IsDBNull(4) ? null : reader.GetString(4));
                    var sourceSiteId = reader.IsDBNull(5) ? null : reader.GetString(5);
                    var sourceInstanceId = reader.IsDBNull(6) ? null : reader.GetString(6);
                    DateTime? firstOccurred = reader.IsDBNull(7) ? null : reader.GetDateTime(7);
                    DateTime? lastOccurred = reader.IsDBNull(8) ? null : reader.GetDateTime(8);

                    nodes.Add(new ExecutionTreeNode(
                        ExecutionId: nodeExecutionId,
                        ParentExecutionId: parentExecutionId,
                        RowCount: rowCount,
                        Channels: channels,
                        Statuses: statuses,
                        SourceSiteId: sourceSiteId,
                        SourceInstanceId: sourceInstanceId,
                        FirstOccurredAtUtc: firstOccurred,
                        LastOccurredAtUtc: lastOccurred));
                }
            }

            return nodes;
        }
        finally
        {
            if (openedHere)
            {
                await conn.CloseAsync().ConfigureAwait(false);
            }
        }
    }

    /// <inheritdoc />
    public async Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default)
    {
        return await _context.Set<AuditLogRow>()
            .AsNoTracking()
            .Where(e => e.SourceNode != null)
            .Select(e => e.SourceNode!)
            .Distinct()
            .OrderBy(n => n)
            .ToListAsync(ct);
    }

    /// <inheritdoc />
    public async Task<long> BackfillSourceNodeAsync(
        string sentinel,
        DateTime before,
        int batchSize,
        CancellationToken ct = default)
    {
        if (string.IsNullOrWhiteSpace(sentinel))
        {
            throw new ArgumentException("Sentinel must be a non-empty value.", nameof(sentinel));
        }

        if (batchSize <= 0)
        {
            throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be > 0.");
        }

        var beforeUtc = DateTime.SpecifyKind(before.ToUniversalTime(), DateTimeKind.Utc);

        // SourceNode sentinel backfill. This is the ONE sanctioned UPDATE
        // against dbo.AuditLog in the codebase. It touches ONLY rows where
        // SourceNode IS NULL AND OccurredAtUtc < @before — rows that pre-date the
        // SourceNode feature and whose node-of-origin is UNKNOWABLE. The sentinel (default
        // "unknown") makes that explicit. ExecutionId/ParentExecutionId are PERSISTED
        // COMPUTED columns derived from DetailsJson — mutating DetailsJson is forbidden
        // under the append-only invariant, so those stay NULL on pre-feature rows.
        //
        // Maintenance path (NOT the writer role): runs on the same connection used for
        // SwitchOutPartitionAsync (partition-switch DDL), which requires a role that
        // holds UPDATE — the append-only scadabridge_audit_writer role has only
        // INSERT + SELECT.
        //
        // Bounded + idempotent: UPDATE TOP (@batch) caps the log/lock footprint per
        // statement; the loop exits when a batch updates 0 rows. Re-running after a
        // crash simply resumes where it left off.
        //
        // The trailing AUDIT-PURGE-ALLOWED marker on the UPDATE line below is the
        // single narrow exemption the append-only CI guard (AuditLogAppendOnlyGuardTests)
        // recognises for an UPDATE; any other UPDATE targeting AuditLog still trips the guard.
        const string updateBatchSql =
            "UPDATE TOP (@batch) dbo.AuditLog SET SourceNode = @sentinel WHERE SourceNode IS NULL AND OccurredAtUtc < @before;"; // AUDIT-PURGE-ALLOWED: SourceNode sentinel backfill, maintenance path

        long totalUpdated = 0;

        var conn = _context.Database.GetDbConnection();
        var openedHere = false;
        if (conn.State != System.Data.ConnectionState.Open)
        {
            await conn.OpenAsync(ct).ConfigureAwait(false);
            openedHere = true;
        }

        try
        {
            while (true)
            {
                ct.ThrowIfCancellationRequested();

                await using var cmd = conn.CreateCommand();
                cmd.CommandText = updateBatchSql;

                var pBatch = cmd.CreateParameter();
                pBatch.ParameterName = "@batch";
                pBatch.Value = batchSize;
                cmd.Parameters.Add(pBatch);

                var pSentinel = cmd.CreateParameter();
                pSentinel.ParameterName = "@sentinel";
                pSentinel.Value = sentinel;
                cmd.Parameters.Add(pSentinel);

                var pBefore = cmd.CreateParameter();
                pBefore.ParameterName = "@before";
                pBefore.Value = beforeUtc;
                cmd.Parameters.Add(pBefore);

                var rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
                if (rows <= 0)
                {
                    break;
                }

                totalUpdated += rows;
            }
        }
        finally
        {
            if (openedHere)
            {
                await conn.CloseAsync().ConfigureAwait(false);
            }
        }

        return totalUpdated;
    }

    /// <summary>
    /// Splits a <c>STRING_AGG</c> comma-joined value into a distinct, ordered
    /// list. A null/empty aggregate (a stub node with no rows) yields an empty
    /// list rather than a single empty string.
    /// </summary>
    private static IReadOnlyList<string> SplitAggregate(string? aggregate)
    {
        if (string.IsNullOrEmpty(aggregate))
        {
            return Array.Empty<string>();
        }

        return aggregate
            .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
            .Distinct(StringComparer.Ordinal)
            .OrderBy(v => v, StringComparer.Ordinal)
            .ToArray();
    }
}