feat(audit): archive retention and superseded stale-archive cleanup (ScadaBridge#109)

Owner decisions recorded: the archive destination stays a directory/UNC
(no object-storage client; share + permissions named at enablement),
parquet-tools-only reading (no re-import), unsigned like the chain (#107).

AuditLog:Archive:RetentionDays (default 0 = never delete; else must exceed
AuditLog:RetentionDays, counted from the archived month's end) and
DeleteSupersededStaleArchives (default true) run after every purge tick.
A retried purge's stale Archived archive is deleted only when a later,
verifying, Purged archive of the same partition provably holds every row:
same prior anchor and the stale chain-end hash at its chain-end sequence
(row or tombstone), every unchained row present by EventId; otherwise it is
reported KeptStale. A sole copy left at Archived is never touched. Deletion
removes only the archive name's own three files in its folder, manifest
first, never a manifest-supplied path. `audit archive cleanup [--dry-run]`
= CleanupAuditArchives (Administrator).

AuditLogArchiveCleanupTests (MSSQL) cover retried purge, kept stale copy,
sole copy, retention and the path guard; each safety rule mutation-checked.
Follow-ups filed: #113, #114, #115.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 23:32:53 -05:00
parent c4d7ec9b96
commit b95ee5a658
26 changed files with 1099 additions and 26 deletions
@@ -32,5 +32,34 @@ public sealed class AuditLogArchiveOptionsValidator : OptionsValidatorBase<Audit
$"AuditLog:Archive:{nameof(AuditLogArchiveOptions.ReadCommandTimeoutMinutes)} ({options.ReadCommandTimeoutMinutes}) must be > 0.");
builder.RequireThat(options.ChainLockTimeoutSeconds > 0,
$"AuditLog:Archive:{nameof(AuditLogArchiveOptions.ChainLockTimeoutSeconds)} ({options.ChainLockTimeoutSeconds}) must be > 0.");
builder.RequireThat(options.RetentionDays is >= 0 and <= 36_500,
$"AuditLog:Archive:{nameof(AuditLogArchiveOptions.RetentionDays)} ({options.RetentionDays}) must be between 0 (never delete) and 36500.");
}
}
/// <summary>
/// Cross-section check for <see cref="AuditLogArchiveOptions.RetentionDays"/> (ScadaBridge#109):
/// when set, it must exceed <c>AuditLog:RetentionDays</c>, so an archive is never deleted before
/// (or as soon as) its rows leave the database — the archive exists to outlive them.
/// </summary>
public sealed class AuditLogArchiveRetentionValidator : Microsoft.Extensions.Options.IValidateOptions<AuditLogArchiveOptions>
{
private readonly Microsoft.Extensions.Options.IOptions<Configuration.AuditLogOptions> _auditOptions;
/// <summary>Creates the validator.</summary>
/// <param name="auditOptions">The <c>AuditLog</c> options (global retention).</param>
public AuditLogArchiveRetentionValidator(Microsoft.Extensions.Options.IOptions<Configuration.AuditLogOptions> auditOptions)
=> _auditOptions = auditOptions ?? throw new ArgumentNullException(nameof(auditOptions));
/// <inheritdoc />
public Microsoft.Extensions.Options.ValidateOptionsResult Validate(string? name, AuditLogArchiveOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var global = _auditOptions.Value.RetentionDays;
return options.RetentionDays == 0 || options.RetentionDays > global
? Microsoft.Extensions.Options.ValidateOptionsResult.Success
: Microsoft.Extensions.Options.ValidateOptionsResult.Fail(
$"AuditLog:Archive:RetentionDays ({options.RetentionDays}) must be 0 (never delete) or greater than "
+ $"AuditLog:RetentionDays ({global}): archives must outlive the rows' time in the database.");
}
}
@@ -6,6 +6,7 @@ using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
@@ -195,6 +196,7 @@ public class AuditLogPurgeActor : ReceiveActor
// No whole-month partitions are eligible, but per-channel overrides may
// still expire rows earlier than the global window — run them below.
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
await RunArchiveCleanupAsync(scope.ServiceProvider, eventStream, purgeFailureCounter).ConfigureAwait(false);
return;
}
@@ -274,6 +276,52 @@ public class AuditLogPurgeActor : ReceiveActor
// deletes only ever expire rows EARLIER than that, so they run last and are a
// strict tightening.
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
// Archive housekeeping last (ScadaBridge#109): superseded stale archives this or an
// earlier tick's retried purge left behind, and archives past AuditLog:Archive:RetentionDays.
if (archive is not null)
{
await RunArchiveCleanupAsync(scope.ServiceProvider, eventStream, purgeFailureCounter, archive).ConfigureAwait(false);
}
}
/// <summary>
/// Runs the archive cleanup pass (ScadaBridge#109) when archiving is enabled. Contained: a
/// cleanup failure is a purge failure on the health surface, never a crash of the tick.
/// </summary>
private async Task RunArchiveCleanupAsync(
IServiceProvider services,
Akka.Event.EventStream eventStream,
IAuditPurgeFailureCounter? purgeFailureCounter,
IAuditLogArchive? archive = null)
{
var sw = Stopwatch.StartNew();
try
{
if (archive is null)
{
if (services.GetService<IOptions<AuditLogArchiveOptions>>()?.Value.Enabled != true)
{
return;
}
archive = services.GetKeyedService<IAuditLogArchive>(AuditMaintenanceContext.ServiceKey)
?? services.GetRequiredService<IAuditLogArchive>();
}
var result = await archive.CleanupAsync(dryRun: false).ConfigureAwait(false);
var deleted = result.Items.Count(i => i.Action != AuditArchiveCleanupAction.KeptStale);
if (deleted > 0)
{
_logger.LogInformation("AuditLog archive cleanup deleted {Count} archive(s) in {DurationMs} ms.", deleted, sw.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
sw.Stop();
_logger.LogError(ex, "AuditLog archive cleanup failed; archives are kept and the next tick retries.");
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, sw.ElapsedMilliseconds);
}
}
/// <summary>
@@ -442,6 +442,10 @@ public static class ServiceCollectionExtensions
// registered by AddConfigurationDatabase (central only) and reads these options.
services.AddValidatedOptions<ConfigurationDatabase.Maintenance.AuditLogArchiveOptions, AuditLogArchiveOptionsValidator>(
config, ArchiveSectionName);
// Archive retention must exceed the global AuditLog:RetentionDays (ScadaBridge#109).
services.TryAddEnumerable(ServiceDescriptor.Singleton<
Microsoft.Extensions.Options.IValidateOptions<ConfigurationDatabase.Maintenance.AuditLogArchiveOptions>,
AuditLogArchiveRetentionValidator>());
// I1 (review): bind the two central-singleton options HERE rather than in
// AddAuditLogCentralReconciliationClient. AkkaHostedService.RegisterCentralActors
@@ -109,6 +109,43 @@ public static class AuditArchiveHelpers
return verified ? 0 : NotVerifiedExitCode;
}
/// <summary>
/// Renders a <c>CleanupAuditArchives</c> result (ScadaBridge#109). Always exit 0 — kept
/// stale archives are reported, not an error.
/// </summary>
/// <param name="json">The success body (an <c>AuditArchiveCleanupResult</c>, camelCase).</param>
/// <param name="format">Resolved output format.</param>
/// <param name="output">Writer for the report.</param>
/// <returns><c>0</c>.</returns>
public static int RenderCleanup(string json, string format, TextWriter output)
{
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
{
output.WriteLine(json);
output.Flush();
return 0;
}
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var dryRun = root.TryGetProperty("dryRun", out var d) && d.ValueKind == JsonValueKind.True;
var retention = Num(root, "retentionDays");
output.WriteLine($"{(dryRun ? "Dry run — nothing deleted. " : "")}Archive retention: {(retention == "0" ? "never delete" : retention + " days")}");
var items = root.TryGetProperty("items", out var i) && i.ValueKind == JsonValueKind.Array ? i.EnumerateArray().ToList() : [];
if (items.Count == 0)
{
output.WriteLine("Nothing to clean up.");
}
foreach (var item in items)
{
output.WriteLine($"{Str(item, "action"),-18} {Str(item, "name")} {Str(item, "detail")}");
}
output.Flush();
return 0;
}
private static string Str(JsonElement e, string name, string missing = "")
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? missing : missing;
@@ -11,9 +11,9 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
/// <c>verify-heads</c>, which checks off-box exported chain heads with
/// <c>VerifyAuditChainHeads</c> (ScadaBridge#107), and <c>archive list</c> / <c>archive verify</c>,
/// which send <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c> (Parquet cold archive,
/// ScadaBridge#42).
/// <c>VerifyAuditChainHeads</c> (ScadaBridge#107), and <c>archive list</c> / <c>archive verify</c>
/// / <c>archive cleanup</c>, which send <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c>
/// (Parquet cold archive, ScadaBridge#42) / <c>CleanupAuditArchives</c> (ScadaBridge#109).
/// </summary>
public static class AuditCommands
{
@@ -376,7 +376,7 @@ public static class AuditCommands
private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
{
var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" };
var group = new Command("archive") { Description = "List, verify and clean up the AuditLog Parquet cold archives" };
var list = new Command("list") { Description = "List the AuditLog Parquet archives written by the retention purge" };
list.SetAction(async (ParseResult result) =>
@@ -410,6 +410,23 @@ public static class AuditCommands
});
group.Add(verify);
var dryRunOption = new Option<bool>("--dry-run") { Description = "Report what would be deleted; delete nothing" };
var cleanup = new Command("cleanup")
{
Description = "Delete superseded stale archives and archives past AuditLog:Archive:RetentionDays (Administrator)",
};
cleanup.Add(dryRunOption);
cleanup.SetAction(async (ParseResult result) =>
{
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new CleanupAuditArchivesCommand(result.GetValue(dryRunOption)),
timeout: AuditArchiveHelpers.VerifyRequestTimeout,
onSuccess: json => AuditArchiveHelpers.RenderCleanup(json, format, Console.Out));
});
group.Add(cleanup);
return group;
}
+7
View File
@@ -2136,6 +2136,13 @@ tombstones, and compares the chain end with the checkpoint anchor the purge left
| `1` | Request error (`ARCHIVE_NOT_CONFIGURED`, `NOT_FOUND`, `INVALID_ARGUMENT`, server error) |
| `2` | Authorization failure (needs `Administrator` or `Viewer`, not site-scoped) |
`audit archive cleanup [--dry-run]` (ScadaBridge#109, Administrator) sends
`CleanupAuditArchives`: it deletes stale archives a retried purge left behind once a later
purged archive provably holds all their rows (`DeletedSuperseded`), and archives whose month
ended more than `AuditLog:Archive:RetentionDays` ago (`DeletedExpired`; default `0` = never),
and lists stale archives it kept (`KeptStale`) with the reason. `--dry-run` deletes nothing.
The purge actor runs the same pass after every purge tick. Exit code `0`.
The archive files are plain Parquet; read them with any Parquet tool, e.g. DuckDB:
`SELECT * FROM '<dir>/2025/auditlog-2025-10-….parquet'`. See `Component-AuditLog.md`,
Retention & Purge.
@@ -66,6 +66,17 @@ public interface IAuditLogArchive
/// <returns>The verification result.</returns>
/// <exception cref="FileNotFoundException">No archive has that name.</exception>
Task<AuditArchiveVerificationResult> VerifyAsync(string name, CancellationToken ct = default);
/// <summary>
/// Archive cleanup (ScadaBridge#109): deletes superseded stale archives a retried purge left
/// behind (when <c>AuditLog:Archive:DeleteSupersededStaleArchives</c>), and archives past
/// <c>AuditLog:Archive:RetentionDays</c> (0 = never). Only ever deletes the three files of a
/// well-formed archive inside the configured directory, manifest first.
/// </summary>
/// <param name="dryRun">Report only; delete nothing.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>What was (or would be) deleted, and the stale archives kept.</returns>
Task<AuditArchiveCleanupResult> CleanupAsync(bool dryRun, CancellationToken ct = default);
}
/// <summary>
@@ -57,3 +57,12 @@ public record ListAuditArchivesCommand;
/// </summary>
/// <param name="Name">The archive name, as <c>audit archive list</c> reports it.</param>
public record VerifyAuditArchiveCommand(string Name);
/// <summary>
/// Runs the AuditLog archive cleanup (ScadaBridge#109) now: deletes superseded stale archives
/// a retried purge left behind and archives past <c>AuditLog:Archive:RetentionDays</c> — or,
/// with <paramref name="DryRun"/>, only reports what it would delete. The purge actor runs the
/// same pass after every purge tick. Administrator only (it deletes files).
/// </summary>
/// <param name="DryRun">Report only; delete nothing.</param>
public record CleanupAuditArchivesCommand(bool DryRun);
@@ -145,3 +145,41 @@ public sealed record AuditArchiveVerificationResult(
/// <param name="Enabled">Whether the retention purge is archiving partitions (<c>AuditLog:Archive:Enabled</c>).</param>
/// <param name="Archives">Every archive manifest, oldest partition first.</param>
public sealed record AuditArchiveListResult(bool Enabled, IReadOnlyList<AuditArchiveManifest> Archives);
/// <summary>What archive cleanup did (or, on a dry run, would do) with one archive (ScadaBridge#109).</summary>
[JsonConverter(typeof(JsonStringEnumConverter<AuditArchiveCleanupAction>))]
public enum AuditArchiveCleanupAction
{
/// <summary>
/// Deleted by retention: the archived month ended more than <c>AuditLog:Archive:RetentionDays</c>
/// ago, and the archive is purged (or its partition no longer holds rows).
/// </summary>
DeletedExpired,
/// <summary>
/// Deleted as a superseded stale archive: status <c>Archived</c> (its partition switch never
/// completed — a retried purge), and a LATER, verified, <c>Purged</c> archive of the same
/// partition provably holds every row it holds (same chain through its chain end, every
/// unchained row present by EventId).
/// </summary>
DeletedSuperseded,
/// <summary>
/// A stale <c>Archived</c> archive that was kept: no later purged archive provably covers it,
/// so it may be the only copy of some rows (see Detail).
/// </summary>
KeptStale,
}
/// <summary>One archive considered by cleanup.</summary>
/// <param name="Name">Archive name.</param>
/// <param name="Partition">Archived partition, <c>YYYY-MM</c>.</param>
/// <param name="Action">What was (or would be) done.</param>
/// <param name="Detail">Why.</param>
public sealed record AuditArchiveCleanupItem(string Name, string Partition, AuditArchiveCleanupAction Action, string Detail);
/// <summary>Result of an archive cleanup pass (ScadaBridge#109).</summary>
/// <param name="DryRun">True when nothing was deleted (the items say what would be).</param>
/// <param name="RetentionDays">The archive retention in force (0 = archives never expire).</param>
/// <param name="Items">The archives deleted (or that would be) and the stale ones kept.</param>
public sealed record AuditArchiveCleanupResult(bool DryRun, int RetentionDays, IReadOnlyList<AuditArchiveCleanupItem> Items);
@@ -43,4 +43,19 @@ public sealed class AuditLogArchiveOptions
/// the hash-chain lock before giving up for this tick (default 300).
/// </summary>
public int ChainLockTimeoutSeconds { get; set; } = 300;
/// <summary>
/// Archive retention in days, counted from the END of the archived month (ScadaBridge#109).
/// <c>0</c> (the default) = archives are never deleted — until compliance names a period.
/// When set it must exceed <c>AuditLog:RetentionDays</c>, so an archive always outlives the
/// rows' time in the database. Applied by the purge actor after each tick's partition purge.
/// </summary>
public int RetentionDays { get; set; }
/// <summary>
/// Delete stale archives a retried purge left behind (status <c>Archived</c>) once a later,
/// verified, purged archive of the same partition provably holds all their rows (default
/// true; ScadaBridge#109). A stale archive that cannot be proven redundant is kept.
/// </summary>
public bool DeleteSupersededStaleArchives { get; set; } = true;
}
@@ -0,0 +1,290 @@
using System.Data;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
/// <summary>
/// Archive cleanup (ScadaBridge#109): retention and superseded-stale-archive deletion.
/// </summary>
/// <remarks>
/// <para>
/// <b>Superseded stale archives.</b> A purge whose guard finds the partition changed after it
/// was archived (51002/51003) purges nothing and leaves its archive at status
/// <see cref="AuditArchiveStatus.Archived"/>; the next tick writes a fresh archive and purges
/// with it. The stale one is redundant only if a LATER archive B of the same partition, which
/// is <see cref="AuditArchiveStatus.Purged"/> and verifies, holds everything the stale archive A
/// holds. That is proven, not assumed: for every chain month of A, B starts from the same
/// earlier purge anchor and carries A's chain-end hash at A's chain-end sequence (as a row or a
/// tombstone) — so, the chain being a hash chain B itself verifies, every chained row of A is
/// in B; and every UNCHAINED row of A is found in B (row or tombstone) by EventId. Anything
/// else — notably an unchained row a per-channel purge removed in between, which leaves no
/// tombstone — keeps A (<see cref="AuditArchiveCleanupAction.KeptStale"/>). An archive left at
/// <c>Archived</c> because the node stopped between the switch and the manifest update has no
/// later purged archive and is kept: it is the only copy.
/// </para>
/// <para>
/// <b>Retention.</b> With <see cref="AuditLogArchiveOptions.RetentionDays"/> &gt; 0, an archive
/// whose month ended more than that many days ago is deleted when it is <c>Purged</c>, or —
/// an <c>Archived</c> one — when its partition no longer holds rows in <c>dbo.AuditLog</c>.
/// </para>
/// <para>
/// <b>Safe deletion.</b> Only the three files a well-formed archive name implies
/// (<c>{name}.manifest.json</c>, <c>{name}.parquet</c>, <c>{name}.tombstones.parquet</c>) in the
/// archive's own <c>{Directory}/{YYYY}</c> folder are deleted — never a path read from the
/// manifest — and the manifest goes FIRST, so an interrupted deletion leaves leftovers that are
/// no longer an archive rather than a manifest naming missing files.
/// </para>
/// </remarks>
public sealed partial class AuditLogPartitionArchive
{
/// <summary>
/// Upper bound on unchained rows compared by EventId when proving a stale archive redundant;
/// a larger set keeps the archive rather than spend the memory.
/// </summary>
internal const long MaxUnchainedRowsCompared = 5_000_000;
/// <inheritdoc />
public async Task<AuditArchiveCleanupResult> CleanupAsync(bool dryRun, CancellationToken ct = default)
{
RequireConfigured();
var items = new List<AuditArchiveCleanupItem>();
var manifests = await ListAsync(ct).ConfigureAwait(false);
var deleted = new HashSet<string>(StringComparer.Ordinal);
if (_options.DeleteSupersededStaleArchives)
{
foreach (var stale in manifests.Where(m => m.Status == AuditArchiveStatus.Archived))
{
var later = manifests
.Where(m => m.PartitionBoundaryUtc == stale.PartitionBoundaryUtc
&& m.Status == AuditArchiveStatus.Purged
&& m.ArchivedAtUtc > stale.ArchivedAtUtc)
.OrderBy(m => m.ArchivedAtUtc)
.ToList();
if (later.Count == 0)
{
continue; // not superseded (e.g. the node stopped before marking it Purged) — keep
}
string? reason = null;
AuditArchiveManifest? coveredBy = null;
foreach (var candidate in later)
{
reason = await ProveCoveredAsync(stale, candidate, ct).ConfigureAwait(false);
if (reason is null)
{
coveredBy = candidate;
break;
}
}
if (coveredBy is null)
{
items.Add(new AuditArchiveCleanupItem(stale.Name, stale.Partition, AuditArchiveCleanupAction.KeptStale,
$"No later purged archive provably covers it: {reason}"));
continue;
}
if (!dryRun)
{
DeleteArchiveFiles(stale);
}
deleted.Add(stale.Name);
items.Add(new AuditArchiveCleanupItem(stale.Name, stale.Partition, AuditArchiveCleanupAction.DeletedSuperseded,
$"Superseded by {coveredBy.Name}, which is purged, verifies and holds every row of this archive."));
}
}
if (_options.RetentionDays > 0)
{
var now = _time.GetUtcNow().UtcDateTime;
foreach (var manifest in manifests.Where(m => !deleted.Contains(m.Name)))
{
var monthEnd = AuditChainHasher.MonthOf(manifest.PartitionBoundaryUtc).AddMonths(1);
if (monthEnd.AddDays(_options.RetentionDays) > now)
{
continue;
}
if (manifest.Status != AuditArchiveStatus.Purged
&& await PartitionHasRowsAsync(manifest.PartitionBoundaryUtc, ct).ConfigureAwait(false))
{
continue; // the partition was never switched out; this archive is not a sole copy, nor expired data gone
}
if (!dryRun)
{
DeleteArchiveFiles(manifest);
}
deleted.Add(manifest.Name);
items.Add(new AuditArchiveCleanupItem(manifest.Name, manifest.Partition, AuditArchiveCleanupAction.DeletedExpired,
$"{manifest.Partition} ended {monthEnd:yyyy-MM-dd}, more than {_options.RetentionDays} days ago (AuditLog:Archive:RetentionDays)."));
}
}
foreach (var item in items.Where(i => i.Action != AuditArchiveCleanupAction.KeptStale))
{
_logger.LogInformation("AuditLog archive cleanup{DryRun}: {Action} {Archive} — {Detail}",
dryRun ? " (dry run)" : "", item.Action, item.Name, item.Detail);
}
return new AuditArchiveCleanupResult(dryRun, _options.RetentionDays, items);
}
/// <summary>
/// Proves <paramref name="later"/> holds every row of <paramref name="stale"/>; returns null
/// when it does, else why not.
/// </summary>
private async Task<string?> ProveCoveredAsync(AuditArchiveManifest stale, AuditArchiveManifest later, CancellationToken ct)
{
// Same chain generation and at least as far, for every chain month the stale archive has.
var laterChains = later.Chains.ToDictionary(c => c.Partition, StringComparer.Ordinal);
var targets = new Dictionary<(DateTime Month, long Seq), string>();
foreach (var chain in stale.Chains)
{
if (!laterChains.TryGetValue(chain.Partition, out var next))
{
return $"{later.Name} has no chain for {chain.Partition}.";
}
if (next.PriorPurgedThroughSeq != chain.PriorPurgedThroughSeq || next.LastSeq < chain.LastSeq)
{
return $"{later.Name}'s chain for {chain.Partition} does not extend this archive's (prior anchor or end differs).";
}
if (chain.LastSeq > (chain.PriorPurgedThroughSeq ?? 0))
{
targets[(AuditArchiveParquet.ParseMonth(chain.Partition), chain.LastSeq)] = chain.LastHash.ToLowerInvariant();
}
}
if (stale.UnchainedRowCount > MaxUnchainedRowsCompared)
{
return $"{stale.UnchainedRowCount} unchained rows exceed the {MaxUnchainedRowsCompared} compared by EventId.";
}
// The later archive must itself be intact and tied to the database.
var verification = await VerifyAsync(later.Name, ct).ConfigureAwait(false);
if (!verification.Verified)
{
return $"{later.Name} does not verify ({verification.FileProblem ?? "chain or anchor problem"}).";
}
var directory = ArchiveDirectory(stale);
var unchained = new HashSet<Guid>();
await using (var input = File.OpenRead(Path.Combine(directory, stale.Name + ".parquet")))
{
await foreach (var row in AuditArchiveParquet.ReadRowsAsync(input, ct).ConfigureAwait(false))
{
if (row.ChainSeq is null)
{
unchained.Add(row.EventId);
}
}
}
var laterDirectory = ArchiveDirectory(later);
await using (var input = File.OpenRead(Path.Combine(laterDirectory, later.Name + ".parquet")))
{
await foreach (var row in AuditArchiveParquet.ReadRowsAsync(input, ct).ConfigureAwait(false))
{
unchained.Remove(row.EventId);
if (row.ChainSeq is { } seq
&& targets.TryGetValue((AuditChainHasher.MonthOf(row.OccurredAtUtc), seq), out var expected))
{
if (row.RowHash is null || !string.Equals(Convert.ToHexStringLower(row.RowHash), expected, StringComparison.Ordinal))
{
return $"{later.Name} carries a different hash at the stale archive's chain end.";
}
targets.Remove((AuditChainHasher.MonthOf(row.OccurredAtUtc), seq));
}
}
}
await using (var input = File.OpenRead(Path.Combine(laterDirectory, later.Name + ".tombstones.parquet")))
{
await foreach (var tombstone in AuditArchiveParquet.ReadTombstonesAsync(input, ct).ConfigureAwait(false))
{
unchained.Remove(tombstone.EventId);
if (targets.TryGetValue((tombstone.ChainMonth, tombstone.ChainSeq), out var expected))
{
if (!string.Equals(Convert.ToHexStringLower(tombstone.RowHash), expected, StringComparison.Ordinal))
{
return $"{later.Name} carries a different hash at the stale archive's chain end.";
}
targets.Remove((tombstone.ChainMonth, tombstone.ChainSeq));
}
}
}
if (targets.Count > 0)
{
return $"{later.Name} does not contain the stale archive's chain end.";
}
return unchained.Count == 0
? null
: $"{unchained.Count} unchained row(s) of this archive are not in {later.Name} (e.g. removed by a per-channel purge, which leaves no tombstone for an unchained row).";
}
private async Task<bool> PartitionHasRowsAsync(DateTime boundary, CancellationToken ct)
{
var conn = _context.Database.GetDbConnection();
var openedHere = await AuditLogHashChain.OpenAsync(conn, ct).ConfigureAwait(false);
try
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT CASE WHEN EXISTS (SELECT 1 FROM dbo.AuditLog " +
"WHERE $partition.pf_AuditLog_Month(OccurredAtUtc) = $partition.pf_AuditLog_Month(@boundary)) THEN 1 ELSE 0 END;";
cmd.Parameters.Add(new SqlParameter("@boundary", SqlDbType.DateTime2) { Scale = 7, Value = boundary });
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) == 1;
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
private string ArchiveDirectory(AuditArchiveManifest manifest)
{
if (!IsValidName(manifest.Name))
{
throw new InvalidOperationException($"'{manifest.Name}' is not an archive name.");
}
return Path.Combine(_options.Directory!, manifest.Name.Substring("auditlog-".Length, 4));
}
/// <summary>
/// Deletes exactly the archive's three files, manifest first. File names are derived from
/// the validated archive name, never taken from the manifest.
/// </summary>
private void DeleteArchiveFiles(AuditArchiveManifest manifest)
{
var directory = Path.GetFullPath(ArchiveDirectory(manifest));
var root = Path.GetFullPath(_options.Directory!);
if (!directory.StartsWith(root, StringComparison.Ordinal))
{
throw new InvalidOperationException($"Archive {manifest.Name} resolves outside {root}.");
}
foreach (var file in new[] { ".manifest.json", ".parquet", ".tombstones.parquet" })
{
var path = Path.Combine(directory, manifest.Name + file);
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}
@@ -398,6 +398,8 @@ public class ManagementActor : ReceiveActor
// AuditLog Parquet archives (ScadaBridge#42): audit reads with verify-chain's gate —
// OperationalAudit roles, site-scoped callers refused in the handler.
ListAuditArchivesCommand => OperationalAuditReaders,
// Archive cleanup deletes archive files (ScadaBridge#109): Administrator only.
CleanupAuditArchivesCommand => AdminOnly,
VerifyAuditArchiveCommand => OperationalAuditReaders,
// Read-only queries -- any authenticated user
@@ -574,6 +576,7 @@ public class ManagementActor : ReceiveActor
VerifyAuditChainHeadsCommand cmd => await HandleVerifyAuditChainHeads(sp, cmd, user),
ListAuditArchivesCommand => await HandleListAuditArchives(sp, user),
VerifyAuditArchiveCommand cmd => await HandleVerifyAuditArchive(sp, cmd, user),
CleanupAuditArchivesCommand cmd => await HandleCleanupAuditArchives(sp, cmd, user),
// Health
GetHealthSummaryCommand => HandleGetHealthSummary(sp),
@@ -3340,6 +3343,20 @@ public class ManagementActor : ReceiveActor
}
}
/// <summary>
/// Runs the archive cleanup now (ScadaBridge#109), on the maintenance-connection archive when
/// registered. Dry run reports only.
/// </summary>
private static async Task<object?> HandleCleanupAuditArchives(
IServiceProvider sp, CleanupAuditArchivesCommand cmd, AuthenticatedUser user)
{
RequireAuditArchive(sp, user);
var archive = sp.GetKeyedService<IAuditLogArchive>(
ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.AuditMaintenanceContext.ServiceKey)
?? sp.GetRequiredService<IAuditLogArchive>();
return await archive.CleanupAsync(cmd.DryRun);
}
private static IAuditLogArchive RequireAuditArchive(IServiceProvider sp, AuthenticatedUser user)
{
if (user.PermittedSiteIds.Length > 0
@@ -69,6 +69,8 @@ public static class ManagementEndpoints
// AuditLog archive verification (ScadaBridge#42) hashes and re-chains a whole month's
// Parquet archive.
typeof(VerifyAuditArchiveCommand),
// Archive cleanup proves stale archives redundant by reading Parquet files (ScadaBridge#109).
typeof(CleanupAuditArchivesCommand),
];
/// <summary>