fix(audit): maintenance-login roll-forward/backfill, archive orphan cleanup, head-export health (ScadaBridge#113 #114 #115)

#113: AuditLogPartitionMaintenance (monthly SPLIT RANGE) and the backfill-source-node endpoint now use the AuditLog:MaintenanceConnectionString login when set (unset = unchanged). Migration AuditMaintenanceGrants gives scadabridge_audit_purger ALTER ANY DATASPACE and column-level UPDATE (SourceNode). #114: archive cleanup deletes the writer's own *.tmp files and manifest-less data files older than 24 h inside {Directory}/{YYYY} (DeletedOrphan, dry-run aware). #115: head-export failures bump scadabridge.audit.chain_head_export.failures and Degrade the /health/ready entry audit-chain-head-export (failing or stale > 2 intervals). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 23:47:47 -05:00
parent b95ee5a658
commit 533414a16d
19 changed files with 2943 additions and 16 deletions
@@ -210,6 +210,68 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
        Assert.Empty(Directory.EnumerateFiles(Path.Combine(dir, "2027")));
    }

+    /// <summary>
+    /// ScadaBridge#114: leftovers of an interrupted archive write are removed — but only the
+    /// writer's own temp / manifest-less data files, old enough, in a year folder. Fresh files,
+    /// committed archives and foreign files survive; dry run deletes nothing.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [SkippableFact]
+    public async Task InterruptedWriteOrphans_AreRemoved_OnlyWhenOldAndWriterNamed()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+        var dir = Dir("orphans");
+        var month = Month(2027, 6);
+        await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
+        await DrainAsync();
+        var committed = await ArchiveAsync(dir, month);
+        var year = Directory.CreateDirectory(Path.Combine(dir, "2027")).FullName;
+
+        const string orphanA = "auditlog-2027-05-20270601T000000001Z";
+        const string freshB = "auditlog-2027-05-20270601T000000002Z";
+        var old = DateTime.UtcNow.AddDays(-3);
+        string Make(string name, DateTime? written = null)
+        {
+            var path = Path.Combine(year, name);
+            File.WriteAllText(path, "x");
+            File.SetLastWriteTimeUtc(path, written ?? old);
+            return path;
+        }
+
+        var oldTmpData = Make(orphanA + ".parquet.tmp");
+        var oldTmpTomb = Make(orphanA + ".tombstones.parquet.tmp");
+        var oldTmpManifest = Make(orphanA + ".manifest.json.tmp");
+        var oldNoManifestData = Make(orphanA + ".parquet");
+        var oldNoManifestTomb = Make(orphanA + ".tombstones.parquet");
+        var freshTmp = Make(freshB + ".parquet.tmp", DateTime.UtcNow.AddHours(-1));
+        var freshNoManifest = Make(freshB + ".parquet", DateTime.UtcNow.AddHours(-1));
+        var foreign = Make("notes.tmp");
+        var wrongName = Make("auditlog-bad.parquet.tmp");
+        var wrongYear = Make("auditlog-2026-05-20260601T000000003Z.parquet.tmp");
+        Directory.CreateDirectory(Path.Combine(dir, "misc"));
+        var otherDir = Path.Combine(dir, "misc", orphanA + ".parquet.tmp");
+        File.WriteAllText(otherDir, "x");
+        File.SetLastWriteTimeUtc(otherDir, old);
+        var committedFiles = new[] { ManifestPath(dir, committed), Path.Combine(year, committed.Name + ".parquet"), Path.Combine(year, committed.Name + ".tombstones.parquet") };
+        foreach (var f in committedFiles)
+        {
+            File.SetLastWriteTimeUtc(f, old);
+        }
+
+        var removed = new[] { oldTmpData, oldTmpTomb, oldTmpManifest, oldNoManifestData, oldNoManifestTomb };
+
+        var dry = await CleanupAsync(dir, dryRun: true);
+        Assert.Equal(5, dry.Items.Count(i => i.Action == AuditArchiveCleanupAction.DeletedOrphan));
+        Assert.All(removed, f => Assert.True(File.Exists(f)));
+
+        var real = await CleanupAsync(dir, dryRun: false);
+        Assert.Equal(5, real.Items.Count(i => i.Action == AuditArchiveCleanupAction.DeletedOrphan));
+        Assert.All(removed, f => Assert.False(File.Exists(f)));
+        Assert.All(new[] { freshTmp, freshNoManifest, foreign, wrongName, wrongYear, otherDir }.Concat(committedFiles), f => Assert.True(File.Exists(f), f));
+
+        Assert.Empty((await CleanupAsync(dir, dryRun: false)).Items);
+    }
+
    private static readonly JsonSerializerOptions Json = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };

    private sealed class FixedClock(DateTime now) : TimeProvider
@@ -160,6 +160,55 @@ IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];"
        Assert.Same(plainScope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>(), shared.Context);
    }

+    /// <summary>
+    /// ScadaBridge#113: the monthly roll-forward (<c>ALTER PARTITION FUNCTION … SPLIT RANGE</c>) and
+    /// the <c>SourceNode</c> backfill run on the maintenance login — and are refused to the writer
+    /// login — with only the role grants the <c>AuditMaintenanceGrants</c> migration provisions.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [SkippableFact]
+    public async Task RollForwardAndSourceNodeBackfill_RunOnTheMaintenanceLogin_NotTheWriter()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+        var writerCs = As(WriterLogin);
+        var maintenanceCs = As(MaintenanceLogin);
+        var month = new DateTime(2027, 4, 1, 0, 0, 0, DateTimeKind.Utc);
+
+        // A pre-feature row: no SourceNode, not chained.
+        await using (var writerCtx = Context(writerCs))
+        {
+            var legacy = NewEvent(month.AddDays(1), AuditChannel.ApiOutbound) with { SourceNode = null };
+            await new AuditLogRepository(writerCtx).InsertIfNotExistsAsync(legacy);
+        }
+
+        // Negative controls: the writer login can neither split the partition function nor backfill.
+        await using (var writerCtx = Context(writerCs))
+        {
+            await Assert.ThrowsAsync<SqlException>(() => new AuditLogPartitionMaintenance(writerCtx).EnsureLookaheadAsync(60));
+            await Assert.ThrowsAsync<SqlException>(() =>
+                new AuditLogRepository(writerCtx).BackfillSourceNodeAsync("unknown", month.AddYears(1), 100));
+        }
+
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddConfigurationDatabase(writerCs, maintenanceCs);
+        await using var provider = services.BuildServiceProvider();
+        await using (var scope = provider.CreateAsyncScope())
+        {
+            var maintenance = scope.ServiceProvider.GetRequiredService<AuditMaintenanceContext>();
+            var roll = scope.ServiceProvider.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.IPartitionMaintenance>();
+            var before = await roll.GetMaxBoundaryAsync();
+            var added = await roll.EnsureLookaheadAsync(60);
+            Assert.NotEmpty(added);
+            Assert.True(await roll.GetMaxBoundaryAsync() > before);
+
+            var repo = scope.ServiceProvider.GetRequiredKeyedService<IAuditLogRepository>(
+                ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.AuditLogServiceKeys.Maintenance);
+            Assert.Equal(1, await repo.BackfillSourceNodeAsync("unknown", month.AddYears(1), 100));
+            Assert.Equal(MaintenanceLogin, await LoginNameAsync(maintenance.Context));
+        }
+    }
+
    private string As(string login) => new SqlConnectionStringBuilder(_fixture.ConnectionString)
    {
        UserID = login,