fix(audit): maintenance-login roll-forward/backfill, archive orphan cleanup, head-export health (ScadaBridge#113 #114 #115)

#113: AuditLogPartitionMaintenance (monthly SPLIT RANGE) and the
backfill-source-node endpoint now use the AuditLog:MaintenanceConnectionString
login when set (unset = unchanged). Migration AuditMaintenanceGrants gives
scadabridge_audit_purger ALTER ANY DATASPACE and column-level UPDATE (SourceNode).
#114: archive cleanup deletes the writer's own *.tmp files and manifest-less
data files older than 24 h inside {Directory}/{YYYY} (DeletedOrphan, dry-run aware).
#115: head-export failures bump scadabridge.audit.chain_head_export.failures and
Degrade the /health/ready entry audit-chain-head-export (failing or stale > 2 intervals).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportService.cs

@@ -31,6 +31,7 @@ public sealed class AuditChainHeadExportService : IHostedService, IDisposable
     private readonly string? _node;
     private readonly TimeProvider _time;
     private readonly ILogger<AuditChainHeadExportService> _logger;
+    private readonly AuditChainHeadExportStatus _status;
     private readonly Dictionary<DateTime, (long Seq, long? Purged, long Links)> _lastExported = new();
     private CancellationTokenSource? _cts;
     private Task? _loop;
@@ -42,14 +43,17 @@ public sealed class AuditChainHeadExportService : IHostedService, IDisposable
     /// <param name="nodeName">This node's name for the records.</param>
     /// <param name="logger">Logger.</param>
     /// <param name="time">Clock.</param>
+    /// <param name="status">Health status the tick outcome is recorded on (ScadaBridge#115).</param>
     public AuditChainHeadExportService(
         IServiceScopeFactory scopeFactory,
         IOptions<AuditLogHashChainOptions> options,
         IAuditChainHeadSink? sink,
         string? nodeName,
         ILogger<AuditChainHeadExportService> logger,
-        TimeProvider? time = null)
+        TimeProvider? time = null,
+        AuditChainHeadExportStatus? status = null)
     {
+        _status = status ?? new AuditChainHeadExportStatus();
         _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value.HeadExport;
         _sink = sink;
@@ -67,6 +71,7 @@ public sealed class AuditChainHeadExportService : IHostedService, IDisposable
             return Task.CompletedTask;
         }
 
+        _status.MarkStarted(_options.IntervalMinutes, _time.GetUtcNow().UtcDateTime);
         _logger.LogInformation("AuditLog chain-head export every {Minutes} min to {Sink}.", _options.IntervalMinutes, _sink.Describe);
         var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
         _cts = cts;
@@ -118,6 +123,7 @@ public sealed class AuditChainHeadExportService : IHostedService, IDisposable
                 .ToList();
             if (changed.Count == 0)
             {
+                _status.RecordSuccess(now);
                 return 0;
             }
 
@@ -128,6 +134,7 @@ public sealed class AuditChainHeadExportService : IHostedService, IDisposable
                 _lastExported[h.Month] = (h.LastSeq, h.PurgedThroughSeq, h.ChainedRows + h.Tombstones);
             }
 
+            _status.RecordSuccess(now);
             _logger.LogDebug("Exported {Count} AuditLog chain head(s) to {Sink}.", records.Count, _sink.Describe);
             return records.Count;
         }
@@ -139,6 +146,7 @@ public sealed class AuditChainHeadExportService : IHostedService, IDisposable
         {
             // Contained: the next tick retries the same heads (nothing was marked exported).
             _logger.LogError(ex, "AuditLog chain-head export to {Sink} failed; retrying next tick.", _sink.Describe);
+            _status.RecordFailure(_time.GetUtcNow().UtcDateTime, ex.Message);
             return 0;
         }
     }

src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportStatus.cs

@@ -0,0 +1,91 @@
+using System.Diagnostics.Metrics;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
+
+namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
+
+/// <summary>
+/// Health signal of the off-box chain-head export (ScadaBridge#115). Updated by
+/// <see cref="AuditChainHeadExportService"/> on every tick; read by the Host's
+/// <c>audit-chain-head-export</c> /health/ready check. A sink that stays down used to be a
+/// log line only, silently stopping the external anchor.
+/// </summary>
+public sealed class AuditChainHeadExportStatus
+{
+    private static readonly Meter Meter = new(ScadaBridgeTelemetry.MeterName);
+
+    private static readonly Counter<long> FailuresCounter =
+        Meter.CreateCounter<long>("scadabridge.audit.chain_head_export.failures", unit: "1",
+            description: "AuditLog chain-head export ticks that failed to read or write the heads (sink offline / unreachable); retried next tick.");
+
+    private readonly object _gate = new();
+    private long _totalFailures;
+    private int _consecutiveFailures;
+    private DateTime? _lastSuccessUtc;
+    private DateTime? _lastFailureUtc;
+    private string? _lastError;
+    private DateTime? _startedUtc;
+
+    /// <summary>Whether the export is running at all (false = disabled, nothing to report).</summary>
+    public bool Enabled { get; private set; }
+
+    /// <summary>Minutes between export ticks (for the staleness test).</summary>
+    public int IntervalMinutes { get; private set; } = 60;
+
+    /// <summary>Total failed ticks since process start.</summary>
+    public long TotalFailures { get { lock (_gate) { return _totalFailures; } } }
+
+    /// <summary>Failed ticks since the last success.</summary>
+    public int ConsecutiveFailures { get { lock (_gate) { return _consecutiveFailures; } } }
+
+    /// <summary>UTC time of the last successful tick, or null.</summary>
+    public DateTime? LastSuccessUtc { get { lock (_gate) { return _lastSuccessUtc; } } }
+
+    /// <summary>UTC time of the last failed tick, or null.</summary>
+    public DateTime? LastFailureUtc { get { lock (_gate) { return _lastFailureUtc; } } }
+
+    /// <summary>Message of the last failure (never a secret: the exception message only), or null.</summary>
+    public string? LastError { get { lock (_gate) { return _lastError; } } }
+
+    /// <summary>UTC time the export loop started, or null.</summary>
+    public DateTime? StartedUtc { get { lock (_gate) { return _startedUtc; } } }
+
+    /// <summary>Marks the export as running.</summary>
+    /// <param name="intervalMinutes">Tick interval.</param>
+    /// <param name="nowUtc">Start time.</param>
+    public void MarkStarted(int intervalMinutes, DateTime nowUtc)
+    {
+        lock (_gate)
+        {
+            Enabled = true;
+            IntervalMinutes = intervalMinutes;
+            _startedUtc = nowUtc;
+        }
+    }
+
+    /// <summary>Records a successful tick.</summary>
+    /// <param name="nowUtc">Time of the tick.</param>
+    public void RecordSuccess(DateTime nowUtc)
+    {
+        lock (_gate)
+        {
+            _consecutiveFailures = 0;
+            _lastSuccessUtc = nowUtc;
+        }
+    }
+
+    /// <summary>Records a failed tick and bumps the failure counter metric.</summary>
+    /// <param name="nowUtc">Time of the tick.</param>
+    /// <param name="error">The failure message.</param>
+    public void RecordFailure(DateTime nowUtc, string error)
+    {
+        lock (_gate)
+        {
+            _totalFailures++;
+            _consecutiveFailures++;
+            _lastFailureUtc = nowUtc;
+            _lastError = error;
+        }
+
+        FailuresCounter.Add(1);
+    }
+}

src/ZB.MOM.WW.ScadaBridge.AuditLog/ServiceCollectionExtensions.cs

@@ -419,6 +419,7 @@ public static class ServiceCollectionExtensions
 
         // Off-box chain-head export (ScadaBridge#107 item 1, AuditLog:HashChain:HeadExport,
         // default off). The sink is built from the bound options; null when disabled.
+        services.AddSingleton<Central.HeadExport.AuditChainHeadExportStatus>();
         services.AddHostedService(sp =>
         {
             var options = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<AuditLogHashChainOptions>>();
@@ -434,7 +435,8 @@ public static class ServiceCollectionExtensions
                 options,
                 sink,
                 node,
-                sp.GetRequiredService<ILogger<Central.HeadExport.AuditChainHeadExportService>>());
+                sp.GetRequiredService<ILogger<Central.HeadExport.AuditChainHeadExportService>>(),
+                status: sp.GetRequiredService<Central.HeadExport.AuditChainHeadExportStatus>());
         });
 
         // Parquet cold archive (ScadaBridge#42). Default OFF; when enabled the purge actor

src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/AuditLogServiceKeys.cs

@@ -0,0 +1,11 @@
+namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+
+/// <summary>DI keys for the AuditLog maintenance-login registrations (ScadaBridge#107/#113).</summary>
+public static class AuditLogServiceKeys
+{
+    /// <summary>
+    /// Key of the maintenance-login <see cref="IAuditLogRepository"/> / <c>IAuditLogArchive</c>:
+    /// the connection from <c>AuditLog:MaintenanceConnectionString</c>, or the runtime connection when unset.
+    /// </summary>
+    public const string Maintenance = "audit-maintenance";
+}

src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditArchive.cs

@@ -169,6 +169,13 @@ public enum AuditArchiveCleanupAction
     /// so it may be the only copy of some rows (see Detail).
     /// </summary>
     KeptStale,
+
+    /// <summary>
+    /// Deleted as an orphan of an interrupted archive write (ScadaBridge#114): a temp file of the
+    /// archive writer (<c>*.parquet.tmp</c>, <c>*.tombstones.parquet.tmp</c>, <c>*.manifest.json.tmp</c>)
+    /// or a data file whose <c>.manifest.json</c> was never written, older than the safe age.
+    /// </summary>
+    DeletedOrphan,
 }
 
 /// <summary>One archive considered by cleanup.</summary>

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogPartitionArchive.Cleanup.cs

@@ -127,6 +127,8 @@ public sealed partial class AuditLogPartitionArchive
             }
         }
 
+        SweepOrphans(dryRun, items);
+
         foreach (var item in items.Where(i => i.Action != AuditArchiveCleanupAction.KeptStale))
         {
             _logger.LogInformation("AuditLog archive cleanup{DryRun}: {Action} {Archive} — {Detail}",
@@ -136,6 +138,92 @@ public sealed partial class AuditLogPartitionArchive
         return new AuditArchiveCleanupResult(dryRun, _options.RetentionDays, items);
     }
 
+    /// <summary>
+    /// Files younger than this are never treated as orphans: a running archive write may still be
+    /// producing them (ScadaBridge#114).
+    /// </summary>
+    internal static readonly TimeSpan OrphanMinAge = TimeSpan.FromHours(24);
+
+    // Longest suffix first. A file is only ever considered through one of these exact suffixes
+    // on a well-formed archive name, so nothing foreign in the directory can match.
+    private static readonly string[] TempSuffixes = [".manifest.json.tmp", ".tombstones.parquet.tmp", ".parquet.tmp"];
+    private static readonly string[] DataSuffixes = [".tombstones.parquet", ".parquet"];
+
+    /// <summary>
+    /// Removes the leftovers of an interrupted archive write (ScadaBridge#114): the writer's own
+    /// temp files, and data files whose manifest — the commit marker — was never written. Only
+    /// files named exactly like the writer names them, directly inside a four-digit
+    /// <c>{Directory}/{YYYY}</c> folder whose year matches the file's, and last written longer ago
+    /// than <see cref="OrphanMinAge"/>. Reparse points are skipped.
+    /// </summary>
+    private void SweepOrphans(bool dryRun, List<AuditArchiveCleanupItem> items)
+    {
+        var root = _options.Directory!;
+        if (!Directory.Exists(root))
+        {
+            return;
+        }
+
+        var cutoff = _time.GetUtcNow().UtcDateTime - OrphanMinAge;
+        foreach (var yearDir in Directory.EnumerateDirectories(root))
+        {
+            var year = Path.GetFileName(yearDir);
+            if (year.Length != 4 || !year.All(char.IsAsciiDigit) || IsReparsePoint(yearDir))
+            {
+                continue;
+            }
+
+            foreach (var path in Directory.EnumerateFiles(yearDir))
+            {
+                var file = Path.GetFileName(path);
+                var tmpSuffix = TempSuffixes.FirstOrDefault(x => file.EndsWith(x, StringComparison.Ordinal));
+                var isTemp = tmpSuffix is not null;
+                var suffix = tmpSuffix ?? DataSuffixes.FirstOrDefault(x => file.EndsWith(x, StringComparison.Ordinal));
+                if (suffix is null)
+                {
+                    continue;
+                }
+
+                var stem = file[..^suffix.Length];
+                if (!IsValidName(stem) || stem.Substring("auditlog-".Length, 4) != year || IsReparsePoint(path))
+                {
+                    continue;
+                }
+
+                if (!isTemp && File.Exists(Path.Combine(yearDir, stem + ".manifest.json")))
+                {
+                    continue; // a committed archive's file
+                }
+
+                if (File.GetLastWriteTimeUtc(path) > cutoff)
+                {
+                    continue; // possibly still being written
+                }
+
+                if (!dryRun)
+                {
+                    try
+                    {
+                        File.Delete(path);
+                    }
+                    catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+                    {
+                        _logger.LogWarning(ex, "AuditLog archive cleanup could not delete orphan {File}.", file);
+                        continue;
+                    }
+                }
+
+                items.Add(new AuditArchiveCleanupItem(stem, stem.Substring("auditlog-".Length, 7), AuditArchiveCleanupAction.DeletedOrphan,
+                    isTemp
+                        ? $"{file}: temp file of an interrupted archive write, older than {OrphanMinAge.TotalHours:0} h."
+                        : $"{file}: no {stem}.manifest.json (the archive write never committed), older than {OrphanMinAge.TotalHours:0} h."));
+            }
+        }
+    }
+
+    private static bool IsReparsePoint(string path)
+        => (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0;
+
     /// <summary>
     /// Proves <paramref name="later"/> holds every row of <paramref name="stale"/>; returns null
     /// when it does, else why not.

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditMaintenanceContext.cs

@@ -1,5 +1,6 @@
 using Microsoft.AspNetCore.DataProtection;
 using Microsoft.EntityFrameworkCore;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
 
 namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
 
@@ -23,7 +24,7 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
 public sealed class AuditMaintenanceContext : IAsyncDisposable, IDisposable
 {
     /// <summary>DI key of the maintenance-login <c>IAuditLogRepository</c> and <c>IAuditLogArchive</c>.</summary>
-    public const string ServiceKey = "audit-maintenance";
+    public const string ServiceKey = AuditLogServiceKeys.Maintenance;
 
     private readonly bool _owned;
 

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/20261110120000_AuditMaintenanceGrants.cs

@@ -0,0 +1,42 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
+{
+    /// <summary>
+    /// Grants for the two AuditLog maintenance paths that now run on the maintenance login
+    /// (<c>AuditLog:MaintenanceConnectionString</c>, ScadaBridge#113). Permission DDL only.
+    /// <list type="bullet">
+    ///   <item><c>ALTER ANY DATASPACE</c> to <c>scadabridge_audit_purger</c> — the monthly
+    ///   roll-forward's <c>ALTER PARTITION FUNCTION … SPLIT RANGE</c> / <c>ALTER PARTITION SCHEME
+    ///   … NEXT USED</c>; <c>ALTER ON SCHEMA::dbo</c> does not confer it.</item>
+    ///   <item>column-level <c>UPDATE (SourceNode)</c> to the same role — the one-time
+    ///   <c>audit backfill-source-node</c>. Column-scoped on purpose: no other AuditLog column
+    ///   becomes updatable.</item>
+    /// </list>
+    /// </summary>
+    public partial class AuditMaintenanceGrants : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(@"
+IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL
+    EXEC sp_executesql N'CREATE ROLE scadabridge_audit_purger';
+GRANT ALTER ANY DATASPACE TO scadabridge_audit_purger;
+GRANT UPDATE (SourceNode) ON dbo.AuditLog TO scadabridge_audit_purger;"); // AUDIT-PURGE-ALLOWED: role grant for the maintenance principal (SourceNode backfill)
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(@"
+IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NOT NULL
+BEGIN
+    REVOKE ALTER ANY DATASPACE FROM scadabridge_audit_purger;
+    REVOKE UPDATE (SourceNode) ON dbo.AuditLog FROM scadabridge_audit_purger;
+END"); // AUDIT-PURGE-ALLOWED: role grant reversal for the maintenance principal
+        }
+    }
+}

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs

@@ -115,7 +115,11 @@ public static class ServiceCollectionExtensions
         // hosted service. Scoped because the implementation reuses the per-scope
         // ScadaBridgeDbContext for raw-SQL execution; the hosted service opens a
         // fresh scope on each tick (mirrors AuditLogPurgeActor / AuditLogIngestActor).
-        services.AddScoped<IPartitionMaintenance, AuditLogPartitionMaintenance>();
+        // Runs on the maintenance connection when AuditLog:MaintenanceConnectionString is set
+        // (ScadaBridge#113), else on the scope's own context as before.
+        services.AddScoped<IPartitionMaintenance>(sp => new AuditLogPartitionMaintenance(
+            sp.GetRequiredService<AuditMaintenanceContext>().Context,
+            sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogPartitionMaintenance>>()));
 
         // AuditLog maintenance connection (ScadaBridge#107 item 3): the chain job and both purge
         // paths run on AuditMaintenanceContext — the scope's own context unless a separate

src/ZB.MOM.WW.ScadaBridge.Host/Health/AuditChainHeadExportHealthCheck.cs

@@ -0,0 +1,63 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Health;
+
+/// <summary>
+/// /health/ready entry (ScadaBridge#115): Degraded while the off-box AuditLog chain-head export
+/// (<c>AuditLog:HashChain:HeadExport</c>) is failing, or has had no success for more than two
+/// intervals. Degraded maps to HTTP 200, so it shows on the Overview dashboard without taking
+/// the node out of rotation. Healthy when the export is disabled.
+/// </summary>
+public sealed class AuditChainHeadExportHealthCheck : IHealthCheck
+{
+    private readonly AuditChainHeadExportStatus _status;
+    private readonly TimeProvider _time;
+
+    /// <summary>Initializes the check.</summary>
+    /// <param name="status">The export status singleton.</param>
+    /// <param name="time">Clock (defaults to the system clock).</param>
+    public AuditChainHeadExportHealthCheck(AuditChainHeadExportStatus status, TimeProvider? time = null)
+    {
+        _status = status ?? throw new ArgumentNullException(nameof(status));
+        _time = time ?? TimeProvider.System;
+    }
+
+    /// <inheritdoc />
+    public Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context, CancellationToken cancellationToken = default)
+    {
+        if (!_status.Enabled)
+        {
+            return Task.FromResult(HealthCheckResult.Healthy("Chain-head export disabled."));
+        }
+
+        var now = _time.GetUtcNow().UtcDateTime;
+        var data = new Dictionary<string, object>(StringComparer.Ordinal)
+        {
+            ["totalFailures"] = _status.TotalFailures,
+            ["consecutiveFailures"] = _status.ConsecutiveFailures,
+            ["intervalMinutes"] = _status.IntervalMinutes,
+        };
+        if (_status.LastSuccessUtc is { } ok)
+        {
+            data["lastSuccessUtc"] = ok;
+        }
+
+        if (_status.ConsecutiveFailures > 0)
+        {
+            return Task.FromResult(HealthCheckResult.Degraded(
+                $"Chain-head export failing ({_status.ConsecutiveFailures} consecutive): {_status.LastError}", data: data));
+        }
+
+        var limit = TimeSpan.FromMinutes(2L * _status.IntervalMinutes);
+        var reference = _status.LastSuccessUtc ?? _status.StartedUtc;
+        if (reference is { } r && now - r > limit)
+        {
+            return Task.FromResult(HealthCheckResult.Degraded(
+                $"No successful chain-head export for more than 2 intervals (last {(_status.LastSuccessUtc is null ? "never" : _status.LastSuccessUtc.Value.ToString("O"))}).", data: data));
+        }
+
+        return Task.FromResult(HealthCheckResult.Healthy("Chain-head export OK.", data));
+    }
+}

src/ZB.MOM.WW.ScadaBridge.Host/Program.cs

@@ -579,6 +579,12 @@ try
                 "bundle-signing-policy",
                 failureStatus: null,
                 tags: new[] { ZbHealthTags.Ready })
+            // ScadaBridge#115: Degraded (HTTP 200) while the off-box chain-head export is failing
+            // or stale; healthy when the export is disabled.
+            .AddTypeActivatedCheck<AuditChainHeadExportHealthCheck>(
+                "audit-chain-head-export",
+                failureStatus: null,
+                tags: new[] { ZbHealthTags.Ready })
             // Oldest-Up-member check (review 01 [High]), unscoped: every central member competes for
             // the one active slot. Since ZB.MOM.WW.Health 0.3.0 this IS the shared check — it selects
             // by age, so the host-local OldestNodeActiveHealthCheck that existed only to avoid the old

src/ZB.MOM.WW.ScadaBridge.ManagementService/AuditEndpoints.cs

@@ -426,7 +426,10 @@ public static class AuditEndpoints
                 statusCode: 400);
         }
 
-        var repo = context.RequestServices.GetRequiredService<IAuditLogRepository>();
+        // Maintenance login when AuditLog:MaintenanceConnectionString is set (ScadaBridge#113);
+        // otherwise the keyed registration resolves to the runtime connection's context.
+        var repo = context.RequestServices.GetKeyedService<IAuditLogRepository>(AuditLogServiceKeys.Maintenance)
+            ?? context.RequestServices.GetRequiredService<IAuditLogRepository>();
         var rowsUpdated = await repo.BackfillSourceNodeAsync(sentinel, beforeUtc.Value, batchSize, context.RequestAborted);
 
         return Results.Json(new