fix(audit): maintenance-login roll-forward/backfill, archive orphan cleanup, head-export health (ScadaBridge#113 #114 #115)

#113: AuditLogPartitionMaintenance (monthly SPLIT RANGE) and the backfill-source-node endpoint now use the AuditLog:MaintenanceConnectionString login when set (unset = unchanged). Migration AuditMaintenanceGrants gives scadabridge_audit_purger ALTER ANY DATASPACE and column-level UPDATE (SourceNode). #114: archive cleanup deletes the writer's own *.tmp files and manifest-less data files older than 24 h inside {Directory}/{YYYY} (DeletedOrphan, dry-run aware). #115: head-export failures bump scadabridge.audit.chain_head_export.failures and Degrade the /health/ready entry audit-chain-head-export (failing or stale > 2 intervals). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 23:47:47 -05:00
parent b95ee5a658
commit 533414a16d
19 changed files with 2943 additions and 16 deletions
@@ -50,7 +50,8 @@ public sealed class AuditChainHeadExportServiceTests
        new DateTime(2026, month, 1, 0, 0, 0, DateTimeKind.Utc), seq, Enumerable.Repeat((byte)seq, 32).ToArray(),
        null, null, rows, 0, new DateTime(2026, month, 2, 0, 0, 0, DateTimeKind.Utc));

-    private static (AuditChainHeadExportService Service, FakeStore Store, RecordingSink Sink) Create(bool exportUnchanged = false)
+    private static (AuditChainHeadExportService Service, FakeStore Store, RecordingSink Sink) Create(
+        bool exportUnchanged = false, AuditChainHeadExportStatus? status = null, TimeProvider? time = null)
    {
        var store = new FakeStore();
        var sink = new RecordingSink();
@@ -62,7 +63,7 @@ public sealed class AuditChainHeadExportServiceTests
        });
        var service = new AuditChainHeadExportService(
            services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>(),
-            options, sink, "central-a", NullLogger<AuditChainHeadExportService>.Instance);
+            options, sink, "central-a", NullLogger<AuditChainHeadExportService>.Instance, time, status);
        return (service, store, sink);
    }

@@ -120,6 +121,48 @@ public sealed class AuditChainHeadExportServiceTests
        Assert.Equal(10, Assert.Single(Assert.Single(sink.Writes)).LastSeq);
    }

+    /// <summary>
+    /// ScadaBridge#115: a failed tick bumps the failure counter metric and the status
+    /// (consecutive + total + last error); the next success clears the consecutive count.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task FailedTick_RecordsMetricAndStatus_SuccessResetsConsecutive()
+    {
+        var status = new AuditChainHeadExportStatus();
+        var (service, store, sink) = Create(status: status);
+        store.Heads.Add(Head(4, 10, 10));
+        long measured = 0;
+        using var listener = new System.Diagnostics.Metrics.MeterListener();
+        listener.InstrumentPublished = (i, l) =>
+        {
+            if (i.Name == "scadabridge.audit.chain_head_export.failures")
+            {
+                l.EnableMeasurementEvents(i);
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((_, v, _, _) => Interlocked.Add(ref measured, v));
+        listener.Start();
+
+        sink.Fail = true;
+        await service.TickAsync(default);
+        await service.TickAsync(default);
+
+        Assert.Equal(2, Interlocked.Read(ref measured));
+        Assert.Equal(2, status.ConsecutiveFailures);
+        Assert.Equal(2, status.TotalFailures);
+        Assert.Equal("share offline", status.LastError);
+        Assert.Null(status.LastSuccessUtc);
+
+        sink.Fail = false;
+        await service.TickAsync(default);
+
+        Assert.Equal(0, status.ConsecutiveFailures);
+        Assert.Equal(2, status.TotalFailures);
+        Assert.NotNull(status.LastSuccessUtc);
+        Assert.Equal(2, Interlocked.Read(ref measured));
+    }
+
    /// <summary>The validator ignores a disabled export and checks each sink's settings when enabled.</summary>
    /// <param name="configure">Which settings to apply.</param>
    /// <param name="expectedError">A fragment of the expected failure, or null when valid.</param>
@@ -210,6 +210,68 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
        Assert.Empty(Directory.EnumerateFiles(Path.Combine(dir, "2027")));
    }

+    /// <summary>
+    /// ScadaBridge#114: leftovers of an interrupted archive write are removed — but only the
+    /// writer's own temp / manifest-less data files, old enough, in a year folder. Fresh files,
+    /// committed archives and foreign files survive; dry run deletes nothing.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [SkippableFact]
+    public async Task InterruptedWriteOrphans_AreRemoved_OnlyWhenOldAndWriterNamed()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+        var dir = Dir("orphans");
+        var month = Month(2027, 6);
+        await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
+        await DrainAsync();
+        var committed = await ArchiveAsync(dir, month);
+        var year = Directory.CreateDirectory(Path.Combine(dir, "2027")).FullName;
+
+        const string orphanA = "auditlog-2027-05-20270601T000000001Z";
+        const string freshB = "auditlog-2027-05-20270601T000000002Z";
+        var old = DateTime.UtcNow.AddDays(-3);
+        string Make(string name, DateTime? written = null)
+        {
+            var path = Path.Combine(year, name);
+            File.WriteAllText(path, "x");
+            File.SetLastWriteTimeUtc(path, written ?? old);
+            return path;
+        }
+
+        var oldTmpData = Make(orphanA + ".parquet.tmp");
+        var oldTmpTomb = Make(orphanA + ".tombstones.parquet.tmp");
+        var oldTmpManifest = Make(orphanA + ".manifest.json.tmp");
+        var oldNoManifestData = Make(orphanA + ".parquet");
+        var oldNoManifestTomb = Make(orphanA + ".tombstones.parquet");
+        var freshTmp = Make(freshB + ".parquet.tmp", DateTime.UtcNow.AddHours(-1));
+        var freshNoManifest = Make(freshB + ".parquet", DateTime.UtcNow.AddHours(-1));
+        var foreign = Make("notes.tmp");
+        var wrongName = Make("auditlog-bad.parquet.tmp");
+        var wrongYear = Make("auditlog-2026-05-20260601T000000003Z.parquet.tmp");
+        Directory.CreateDirectory(Path.Combine(dir, "misc"));
+        var otherDir = Path.Combine(dir, "misc", orphanA + ".parquet.tmp");
+        File.WriteAllText(otherDir, "x");
+        File.SetLastWriteTimeUtc(otherDir, old);
+        var committedFiles = new[] { ManifestPath(dir, committed), Path.Combine(year, committed.Name + ".parquet"), Path.Combine(year, committed.Name + ".tombstones.parquet") };
+        foreach (var f in committedFiles)
+        {
+            File.SetLastWriteTimeUtc(f, old);
+        }
+
+        var removed = new[] { oldTmpData, oldTmpTomb, oldTmpManifest, oldNoManifestData, oldNoManifestTomb };
+
+        var dry = await CleanupAsync(dir, dryRun: true);
+        Assert.Equal(5, dry.Items.Count(i => i.Action == AuditArchiveCleanupAction.DeletedOrphan));
+        Assert.All(removed, f => Assert.True(File.Exists(f)));
+
+        var real = await CleanupAsync(dir, dryRun: false);
+        Assert.Equal(5, real.Items.Count(i => i.Action == AuditArchiveCleanupAction.DeletedOrphan));
+        Assert.All(removed, f => Assert.False(File.Exists(f)));
+        Assert.All(new[] { freshTmp, freshNoManifest, foreign, wrongName, wrongYear, otherDir }.Concat(committedFiles), f => Assert.True(File.Exists(f), f));
+
+        Assert.Empty((await CleanupAsync(dir, dryRun: false)).Items);
+    }
+
    private static readonly JsonSerializerOptions Json = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };

    private sealed class FixedClock(DateTime now) : TimeProvider
@@ -160,6 +160,55 @@ IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];"
        Assert.Same(plainScope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>(), shared.Context);
    }

+    /// <summary>
+    /// ScadaBridge#113: the monthly roll-forward (<c>ALTER PARTITION FUNCTION … SPLIT RANGE</c>) and
+    /// the <c>SourceNode</c> backfill run on the maintenance login — and are refused to the writer
+    /// login — with only the role grants the <c>AuditMaintenanceGrants</c> migration provisions.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [SkippableFact]
+    public async Task RollForwardAndSourceNodeBackfill_RunOnTheMaintenanceLogin_NotTheWriter()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+        var writerCs = As(WriterLogin);
+        var maintenanceCs = As(MaintenanceLogin);
+        var month = new DateTime(2027, 4, 1, 0, 0, 0, DateTimeKind.Utc);
+
+        // A pre-feature row: no SourceNode, not chained.
+        await using (var writerCtx = Context(writerCs))
+        {
+            var legacy = NewEvent(month.AddDays(1), AuditChannel.ApiOutbound) with { SourceNode = null };
+            await new AuditLogRepository(writerCtx).InsertIfNotExistsAsync(legacy);
+        }
+
+        // Negative controls: the writer login can neither split the partition function nor backfill.
+        await using (var writerCtx = Context(writerCs))
+        {
+            await Assert.ThrowsAsync<SqlException>(() => new AuditLogPartitionMaintenance(writerCtx).EnsureLookaheadAsync(60));
+            await Assert.ThrowsAsync<SqlException>(() =>
+                new AuditLogRepository(writerCtx).BackfillSourceNodeAsync("unknown", month.AddYears(1), 100));
+        }
+
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddConfigurationDatabase(writerCs, maintenanceCs);
+        await using var provider = services.BuildServiceProvider();
+        await using (var scope = provider.CreateAsyncScope())
+        {
+            var maintenance = scope.ServiceProvider.GetRequiredService<AuditMaintenanceContext>();
+            var roll = scope.ServiceProvider.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.IPartitionMaintenance>();
+            var before = await roll.GetMaxBoundaryAsync();
+            var added = await roll.EnsureLookaheadAsync(60);
+            Assert.NotEmpty(added);
+            Assert.True(await roll.GetMaxBoundaryAsync() > before);
+
+            var repo = scope.ServiceProvider.GetRequiredKeyedService<IAuditLogRepository>(
+                ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.AuditLogServiceKeys.Maintenance);
+            Assert.Equal(1, await repo.BackfillSourceNodeAsync("unknown", month.AddYears(1), 100));
+            Assert.Equal(MaintenanceLogin, await LoginNameAsync(maintenance.Context));
+        }
+    }
+
    private string As(string login) => new SqlConnectionStringBuilder(_fixture.ConnectionString)
    {
        UserID = login,
@@ -0,0 +1,45 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
+using ZB.MOM.WW.ScadaBridge.Host.Health;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
+
+/// <summary>ScadaBridge#115: the chain-head export readiness entry.</summary>
+public sealed class AuditChainHeadExportHealthCheckTests
+{
+    private sealed class Clock(DateTime now) : TimeProvider
+    {
+        public DateTime Now { get; set; } = now;
+        public override DateTimeOffset GetUtcNow() => new(DateTime.SpecifyKind(Now, DateTimeKind.Utc));
+    }
+
+    private static readonly DateTime T0 = new(2026, 11, 10, 12, 0, 0, DateTimeKind.Utc);
+
+    private static async Task<HealthStatus> Check(AuditChainHeadExportStatus s, Clock c)
+        => (await new AuditChainHeadExportHealthCheck(s, c).CheckHealthAsync(new HealthCheckContext())).Status;
+
+    /// <summary>Disabled export is healthy; failing or stale export is Degraded; recovery clears it.</summary>
+    [Fact]
+    public async Task Status_follows_export_outcome()
+    {
+        var clock = new Clock(T0);
+        var status = new AuditChainHeadExportStatus();
+        Assert.Equal(HealthStatus.Healthy, await Check(status, clock)); // disabled
+
+        status.MarkStarted(60, T0);
+        Assert.Equal(HealthStatus.Healthy, await Check(status, clock));
+
+        status.RecordFailure(T0, "share offline");
+        var result = await new AuditChainHeadExportHealthCheck(status, clock).CheckHealthAsync(new HealthCheckContext());
+        Assert.Equal(HealthStatus.Degraded, result.Status);
+        Assert.Contains("share offline", result.Description);
+
+        status.RecordSuccess(T0);
+        Assert.Equal(HealthStatus.Healthy, await Check(status, clock));
+
+        clock.Now = T0.AddMinutes(121); // no success for > 2 intervals
+        Assert.Equal(HealthStatus.Degraded, await Check(status, clock));
+        status.RecordSuccess(clock.Now);
+        Assert.Equal(HealthStatus.Healthy, await Check(status, clock));
+    }
+}