feat(audit): archive retention and superseded stale-archive cleanup (ScadaBridge#109)
Owner decisions recorded: the archive destination stays a directory/UNC (no object-storage client; share + permissions named at enablement), parquet-tools-only reading (no re-import), unsigned like the chain (#107). AuditLog:Archive:RetentionDays (default 0 = never delete; else must exceed AuditLog:RetentionDays, counted from the archived month's end) and DeleteSupersededStaleArchives (default true) run after every purge tick. A retried purge's stale Archived archive is deleted only when a later, verifying, Purged archive of the same partition provably holds every row: same prior anchor and the stale chain-end hash at its chain-end sequence (row or tombstone), every unchained row present by EventId; otherwise it is reported KeptStale. A sole copy left at Archived is never touched. Deletion removes only the archive name's own three files in its folder, manifest first, never a manifest-supplied path. `audit archive cleanup [--dry-run]` = CleanupAuditArchives (Administrator). AuditLogArchiveCleanupTests (MSSQL) cover retried purge, kept stale copy, sole copy, retention and the path guard; each safety rule mutation-checked. Follow-ups filed: #113, #114, #115. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -32,5 +32,34 @@ public sealed class AuditLogArchiveOptionsValidator : OptionsValidatorBase<Audit
|
||||
$"AuditLog:Archive:{nameof(AuditLogArchiveOptions.ReadCommandTimeoutMinutes)} ({options.ReadCommandTimeoutMinutes}) must be > 0.");
|
||||
builder.RequireThat(options.ChainLockTimeoutSeconds > 0,
|
||||
$"AuditLog:Archive:{nameof(AuditLogArchiveOptions.ChainLockTimeoutSeconds)} ({options.ChainLockTimeoutSeconds}) must be > 0.");
|
||||
builder.RequireThat(options.RetentionDays is >= 0 and <= 36_500,
|
||||
$"AuditLog:Archive:{nameof(AuditLogArchiveOptions.RetentionDays)} ({options.RetentionDays}) must be between 0 (never delete) and 36500.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cross-section check for <see cref="AuditLogArchiveOptions.RetentionDays"/> (ScadaBridge#109):
|
||||
/// when set, it must exceed <c>AuditLog:RetentionDays</c>, so an archive is never deleted before
|
||||
/// (or as soon as) its rows leave the database — the archive exists to outlive them.
|
||||
/// </summary>
|
||||
public sealed class AuditLogArchiveRetentionValidator : Microsoft.Extensions.Options.IValidateOptions<AuditLogArchiveOptions>
|
||||
{
|
||||
private readonly Microsoft.Extensions.Options.IOptions<Configuration.AuditLogOptions> _auditOptions;
|
||||
|
||||
/// <summary>Creates the validator.</summary>
|
||||
/// <param name="auditOptions">The <c>AuditLog</c> options (global retention).</param>
|
||||
public AuditLogArchiveRetentionValidator(Microsoft.Extensions.Options.IOptions<Configuration.AuditLogOptions> auditOptions)
|
||||
=> _auditOptions = auditOptions ?? throw new ArgumentNullException(nameof(auditOptions));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Microsoft.Extensions.Options.ValidateOptionsResult Validate(string? name, AuditLogArchiveOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
var global = _auditOptions.Value.RetentionDays;
|
||||
return options.RetentionDays == 0 || options.RetentionDays > global
|
||||
? Microsoft.Extensions.Options.ValidateOptionsResult.Success
|
||||
: Microsoft.Extensions.Options.ValidateOptionsResult.Fail(
|
||||
$"AuditLog:Archive:RetentionDays ({options.RetentionDays}) must be 0 (never delete) or greater than "
|
||||
+ $"AuditLog:RetentionDays ({global}): archives must outlive the rows' time in the database.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
@@ -195,6 +196,7 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
// No whole-month partitions are eligible, but per-channel overrides may
|
||||
// still expire rows earlier than the global window — run them below.
|
||||
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
|
||||
await RunArchiveCleanupAsync(scope.ServiceProvider, eventStream, purgeFailureCounter).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -274,6 +276,52 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
// deletes only ever expire rows EARLIER than that, so they run last and are a
|
||||
// strict tightening.
|
||||
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
|
||||
|
||||
// Archive housekeeping last (ScadaBridge#109): superseded stale archives this or an
|
||||
// earlier tick's retried purge left behind, and archives past AuditLog:Archive:RetentionDays.
|
||||
if (archive is not null)
|
||||
{
|
||||
await RunArchiveCleanupAsync(scope.ServiceProvider, eventStream, purgeFailureCounter, archive).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the archive cleanup pass (ScadaBridge#109) when archiving is enabled. Contained: a
|
||||
/// cleanup failure is a purge failure on the health surface, never a crash of the tick.
|
||||
/// </summary>
|
||||
private async Task RunArchiveCleanupAsync(
|
||||
IServiceProvider services,
|
||||
Akka.Event.EventStream eventStream,
|
||||
IAuditPurgeFailureCounter? purgeFailureCounter,
|
||||
IAuditLogArchive? archive = null)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
if (archive is null)
|
||||
{
|
||||
if (services.GetService<IOptions<AuditLogArchiveOptions>>()?.Value.Enabled != true)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
archive = services.GetKeyedService<IAuditLogArchive>(AuditMaintenanceContext.ServiceKey)
|
||||
?? services.GetRequiredService<IAuditLogArchive>();
|
||||
}
|
||||
|
||||
var result = await archive.CleanupAsync(dryRun: false).ConfigureAwait(false);
|
||||
var deleted = result.Items.Count(i => i.Action != AuditArchiveCleanupAction.KeptStale);
|
||||
if (deleted > 0)
|
||||
{
|
||||
_logger.LogInformation("AuditLog archive cleanup deleted {Count} archive(s) in {DurationMs} ms.", deleted, sw.ElapsedMilliseconds);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
sw.Stop();
|
||||
_logger.LogError(ex, "AuditLog archive cleanup failed; archives are kept and the next tick retries.");
|
||||
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, sw.ElapsedMilliseconds);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -442,6 +442,10 @@ public static class ServiceCollectionExtensions
|
||||
// registered by AddConfigurationDatabase (central only) and reads these options.
|
||||
services.AddValidatedOptions<ConfigurationDatabase.Maintenance.AuditLogArchiveOptions, AuditLogArchiveOptionsValidator>(
|
||||
config, ArchiveSectionName);
|
||||
// Archive retention must exceed the global AuditLog:RetentionDays (ScadaBridge#109).
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<
|
||||
Microsoft.Extensions.Options.IValidateOptions<ConfigurationDatabase.Maintenance.AuditLogArchiveOptions>,
|
||||
AuditLogArchiveRetentionValidator>());
|
||||
|
||||
// I1 (review): bind the two central-singleton options HERE rather than in
|
||||
// AddAuditLogCentralReconciliationClient. AkkaHostedService.RegisterCentralActors
|
||||
|
||||
@@ -109,6 +109,43 @@ public static class AuditArchiveHelpers
|
||||
return verified ? 0 : NotVerifiedExitCode;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Renders a <c>CleanupAuditArchives</c> result (ScadaBridge#109). Always exit 0 — kept
|
||||
/// stale archives are reported, not an error.
|
||||
/// </summary>
|
||||
/// <param name="json">The success body (an <c>AuditArchiveCleanupResult</c>, camelCase).</param>
|
||||
/// <param name="format">Resolved output format.</param>
|
||||
/// <param name="output">Writer for the report.</param>
|
||||
/// <returns><c>0</c>.</returns>
|
||||
public static int RenderCleanup(string json, string format, TextWriter output)
|
||||
{
|
||||
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
output.WriteLine(json);
|
||||
output.Flush();
|
||||
return 0;
|
||||
}
|
||||
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
var dryRun = root.TryGetProperty("dryRun", out var d) && d.ValueKind == JsonValueKind.True;
|
||||
var retention = Num(root, "retentionDays");
|
||||
output.WriteLine($"{(dryRun ? "Dry run — nothing deleted. " : "")}Archive retention: {(retention == "0" ? "never delete" : retention + " days")}");
|
||||
var items = root.TryGetProperty("items", out var i) && i.ValueKind == JsonValueKind.Array ? i.EnumerateArray().ToList() : [];
|
||||
if (items.Count == 0)
|
||||
{
|
||||
output.WriteLine("Nothing to clean up.");
|
||||
}
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
output.WriteLine($"{Str(item, "action"),-18} {Str(item, "name")} {Str(item, "detail")}");
|
||||
}
|
||||
|
||||
output.Flush();
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static string Str(JsonElement e, string name, string missing = "")
|
||||
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? missing : missing;
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
||||
/// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
|
||||
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
|
||||
/// <c>verify-heads</c>, which checks off-box exported chain heads with
|
||||
/// <c>VerifyAuditChainHeads</c> (ScadaBridge#107), and <c>archive list</c> / <c>archive verify</c>,
|
||||
/// which send <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c> (Parquet cold archive,
|
||||
/// ScadaBridge#42).
|
||||
/// <c>VerifyAuditChainHeads</c> (ScadaBridge#107), and <c>archive list</c> / <c>archive verify</c>
|
||||
/// / <c>archive cleanup</c>, which send <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c>
|
||||
/// (Parquet cold archive, ScadaBridge#42) / <c>CleanupAuditArchives</c> (ScadaBridge#109).
|
||||
/// </summary>
|
||||
public static class AuditCommands
|
||||
{
|
||||
@@ -376,7 +376,7 @@ public static class AuditCommands
|
||||
|
||||
private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
|
||||
{
|
||||
var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" };
|
||||
var group = new Command("archive") { Description = "List, verify and clean up the AuditLog Parquet cold archives" };
|
||||
|
||||
var list = new Command("list") { Description = "List the AuditLog Parquet archives written by the retention purge" };
|
||||
list.SetAction(async (ParseResult result) =>
|
||||
@@ -410,6 +410,23 @@ public static class AuditCommands
|
||||
});
|
||||
group.Add(verify);
|
||||
|
||||
var dryRunOption = new Option<bool>("--dry-run") { Description = "Report what would be deleted; delete nothing" };
|
||||
var cleanup = new Command("cleanup")
|
||||
{
|
||||
Description = "Delete superseded stale archives and archives past AuditLog:Archive:RetentionDays (Administrator)",
|
||||
};
|
||||
cleanup.Add(dryRunOption);
|
||||
cleanup.SetAction(async (ParseResult result) =>
|
||||
{
|
||||
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
|
||||
return await CommandHelpers.ExecuteCommandAsync(
|
||||
result, urlOption, formatOption, usernameOption, passwordOption,
|
||||
new CleanupAuditArchivesCommand(result.GetValue(dryRunOption)),
|
||||
timeout: AuditArchiveHelpers.VerifyRequestTimeout,
|
||||
onSuccess: json => AuditArchiveHelpers.RenderCleanup(json, format, Console.Out));
|
||||
});
|
||||
group.Add(cleanup);
|
||||
|
||||
return group;
|
||||
}
|
||||
|
||||
|
||||
@@ -2136,6 +2136,13 @@ tombstones, and compares the chain end with the checkpoint anchor the purge left
|
||||
| `1` | Request error (`ARCHIVE_NOT_CONFIGURED`, `NOT_FOUND`, `INVALID_ARGUMENT`, server error) |
|
||||
| `2` | Authorization failure (needs `Administrator` or `Viewer`, not site-scoped) |
|
||||
|
||||
`audit archive cleanup [--dry-run]` (ScadaBridge#109, Administrator) sends
|
||||
`CleanupAuditArchives`: it deletes stale archives a retried purge left behind once a later
|
||||
purged archive provably holds all their rows (`DeletedSuperseded`), and archives whose month
|
||||
ended more than `AuditLog:Archive:RetentionDays` ago (`DeletedExpired`; default `0` = never),
|
||||
and lists stale archives it kept (`KeptStale`) with the reason. `--dry-run` deletes nothing.
|
||||
The purge actor runs the same pass after every purge tick. Exit code `0`.
|
||||
|
||||
The archive files are plain Parquet; read them with any Parquet tool, e.g. DuckDB:
|
||||
`SELECT * FROM '<dir>/2025/auditlog-2025-10-….parquet'`. See `Component-AuditLog.md`,
|
||||
Retention & Purge.
|
||||
|
||||
@@ -66,6 +66,17 @@ public interface IAuditLogArchive
|
||||
/// <returns>The verification result.</returns>
|
||||
/// <exception cref="FileNotFoundException">No archive has that name.</exception>
|
||||
Task<AuditArchiveVerificationResult> VerifyAsync(string name, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Archive cleanup (ScadaBridge#109): deletes superseded stale archives a retried purge left
|
||||
/// behind (when <c>AuditLog:Archive:DeleteSupersededStaleArchives</c>), and archives past
|
||||
/// <c>AuditLog:Archive:RetentionDays</c> (0 = never). Only ever deletes the three files of a
|
||||
/// well-formed archive inside the configured directory, manifest first.
|
||||
/// </summary>
|
||||
/// <param name="dryRun">Report only; delete nothing.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>What was (or would be) deleted, and the stale archives kept.</returns>
|
||||
Task<AuditArchiveCleanupResult> CleanupAsync(bool dryRun, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -57,3 +57,12 @@ public record ListAuditArchivesCommand;
|
||||
/// </summary>
|
||||
/// <param name="Name">The archive name, as <c>audit archive list</c> reports it.</param>
|
||||
public record VerifyAuditArchiveCommand(string Name);
|
||||
|
||||
/// <summary>
|
||||
/// Runs the AuditLog archive cleanup (ScadaBridge#109) now: deletes superseded stale archives
|
||||
/// a retried purge left behind and archives past <c>AuditLog:Archive:RetentionDays</c> — or,
|
||||
/// with <paramref name="DryRun"/>, only reports what it would delete. The purge actor runs the
|
||||
/// same pass after every purge tick. Administrator only (it deletes files).
|
||||
/// </summary>
|
||||
/// <param name="DryRun">Report only; delete nothing.</param>
|
||||
public record CleanupAuditArchivesCommand(bool DryRun);
|
||||
|
||||
@@ -145,3 +145,41 @@ public sealed record AuditArchiveVerificationResult(
|
||||
/// <param name="Enabled">Whether the retention purge is archiving partitions (<c>AuditLog:Archive:Enabled</c>).</param>
|
||||
/// <param name="Archives">Every archive manifest, oldest partition first.</param>
|
||||
public sealed record AuditArchiveListResult(bool Enabled, IReadOnlyList<AuditArchiveManifest> Archives);
|
||||
|
||||
/// <summary>What archive cleanup did (or, on a dry run, would do) with one archive (ScadaBridge#109).</summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<AuditArchiveCleanupAction>))]
|
||||
public enum AuditArchiveCleanupAction
|
||||
{
|
||||
/// <summary>
|
||||
/// Deleted by retention: the archived month ended more than <c>AuditLog:Archive:RetentionDays</c>
|
||||
/// ago, and the archive is purged (or its partition no longer holds rows).
|
||||
/// </summary>
|
||||
DeletedExpired,
|
||||
|
||||
/// <summary>
|
||||
/// Deleted as a superseded stale archive: status <c>Archived</c> (its partition switch never
|
||||
/// completed — a retried purge), and a LATER, verified, <c>Purged</c> archive of the same
|
||||
/// partition provably holds every row it holds (same chain through its chain end, every
|
||||
/// unchained row present by EventId).
|
||||
/// </summary>
|
||||
DeletedSuperseded,
|
||||
|
||||
/// <summary>
|
||||
/// A stale <c>Archived</c> archive that was kept: no later purged archive provably covers it,
|
||||
/// so it may be the only copy of some rows (see Detail).
|
||||
/// </summary>
|
||||
KeptStale,
|
||||
}
|
||||
|
||||
/// <summary>One archive considered by cleanup.</summary>
|
||||
/// <param name="Name">Archive name.</param>
|
||||
/// <param name="Partition">Archived partition, <c>YYYY-MM</c>.</param>
|
||||
/// <param name="Action">What was (or would be) done.</param>
|
||||
/// <param name="Detail">Why.</param>
|
||||
public sealed record AuditArchiveCleanupItem(string Name, string Partition, AuditArchiveCleanupAction Action, string Detail);
|
||||
|
||||
/// <summary>Result of an archive cleanup pass (ScadaBridge#109).</summary>
|
||||
/// <param name="DryRun">True when nothing was deleted (the items say what would be).</param>
|
||||
/// <param name="RetentionDays">The archive retention in force (0 = archives never expire).</param>
|
||||
/// <param name="Items">The archives deleted (or that would be) and the stale ones kept.</param>
|
||||
public sealed record AuditArchiveCleanupResult(bool DryRun, int RetentionDays, IReadOnlyList<AuditArchiveCleanupItem> Items);
|
||||
|
||||
@@ -43,4 +43,19 @@ public sealed class AuditLogArchiveOptions
|
||||
/// the hash-chain lock before giving up for this tick (default 300).
|
||||
/// </summary>
|
||||
public int ChainLockTimeoutSeconds { get; set; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Archive retention in days, counted from the END of the archived month (ScadaBridge#109).
|
||||
/// <c>0</c> (the default) = archives are never deleted — until compliance names a period.
|
||||
/// When set it must exceed <c>AuditLog:RetentionDays</c>, so an archive always outlives the
|
||||
/// rows' time in the database. Applied by the purge actor after each tick's partition purge.
|
||||
/// </summary>
|
||||
public int RetentionDays { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Delete stale archives a retried purge left behind (status <c>Archived</c>) once a later,
|
||||
/// verified, purged archive of the same partition provably holds all their rows (default
|
||||
/// true; ScadaBridge#109). A stale archive that cannot be proven redundant is kept.
|
||||
/// </summary>
|
||||
public bool DeleteSupersededStaleArchives { get; set; } = true;
|
||||
}
|
||||
|
||||
+290
@@ -0,0 +1,290 @@
|
||||
using System.Data;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
/// <summary>
|
||||
/// Archive cleanup (ScadaBridge#109): retention and superseded-stale-archive deletion.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Superseded stale archives.</b> A purge whose guard finds the partition changed after it
|
||||
/// was archived (51002/51003) purges nothing and leaves its archive at status
|
||||
/// <see cref="AuditArchiveStatus.Archived"/>; the next tick writes a fresh archive and purges
|
||||
/// with it. The stale one is redundant only if a LATER archive B of the same partition, which
|
||||
/// is <see cref="AuditArchiveStatus.Purged"/> and verifies, holds everything the stale archive A
|
||||
/// holds. That is proven, not assumed: for every chain month of A, B starts from the same
|
||||
/// earlier purge anchor and carries A's chain-end hash at A's chain-end sequence (as a row or a
|
||||
/// tombstone) — so, the chain being a hash chain B itself verifies, every chained row of A is
|
||||
/// in B; and every UNCHAINED row of A is found in B (row or tombstone) by EventId. Anything
|
||||
/// else — notably an unchained row a per-channel purge removed in between, which leaves no
|
||||
/// tombstone — keeps A (<see cref="AuditArchiveCleanupAction.KeptStale"/>). An archive left at
|
||||
/// <c>Archived</c> because the node stopped between the switch and the manifest update has no
|
||||
/// later purged archive and is kept: it is the only copy.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Retention.</b> With <see cref="AuditLogArchiveOptions.RetentionDays"/> > 0, an archive
|
||||
/// whose month ended more than that many days ago is deleted when it is <c>Purged</c>, or —
|
||||
/// an <c>Archived</c> one — when its partition no longer holds rows in <c>dbo.AuditLog</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Safe deletion.</b> Only the three files a well-formed archive name implies
|
||||
/// (<c>{name}.manifest.json</c>, <c>{name}.parquet</c>, <c>{name}.tombstones.parquet</c>) in the
|
||||
/// archive's own <c>{Directory}/{YYYY}</c> folder are deleted — never a path read from the
|
||||
/// manifest — and the manifest goes FIRST, so an interrupted deletion leaves leftovers that are
|
||||
/// no longer an archive rather than a manifest naming missing files.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed partial class AuditLogPartitionArchive
|
||||
{
|
||||
/// <summary>
|
||||
/// Upper bound on unchained rows compared by EventId when proving a stale archive redundant;
|
||||
/// a larger set keeps the archive rather than spend the memory.
|
||||
/// </summary>
|
||||
internal const long MaxUnchainedRowsCompared = 5_000_000;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<AuditArchiveCleanupResult> CleanupAsync(bool dryRun, CancellationToken ct = default)
|
||||
{
|
||||
RequireConfigured();
|
||||
var items = new List<AuditArchiveCleanupItem>();
|
||||
var manifests = await ListAsync(ct).ConfigureAwait(false);
|
||||
var deleted = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
if (_options.DeleteSupersededStaleArchives)
|
||||
{
|
||||
foreach (var stale in manifests.Where(m => m.Status == AuditArchiveStatus.Archived))
|
||||
{
|
||||
var later = manifests
|
||||
.Where(m => m.PartitionBoundaryUtc == stale.PartitionBoundaryUtc
|
||||
&& m.Status == AuditArchiveStatus.Purged
|
||||
&& m.ArchivedAtUtc > stale.ArchivedAtUtc)
|
||||
.OrderBy(m => m.ArchivedAtUtc)
|
||||
.ToList();
|
||||
if (later.Count == 0)
|
||||
{
|
||||
continue; // not superseded (e.g. the node stopped before marking it Purged) — keep
|
||||
}
|
||||
|
||||
string? reason = null;
|
||||
AuditArchiveManifest? coveredBy = null;
|
||||
foreach (var candidate in later)
|
||||
{
|
||||
reason = await ProveCoveredAsync(stale, candidate, ct).ConfigureAwait(false);
|
||||
if (reason is null)
|
||||
{
|
||||
coveredBy = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (coveredBy is null)
|
||||
{
|
||||
items.Add(new AuditArchiveCleanupItem(stale.Name, stale.Partition, AuditArchiveCleanupAction.KeptStale,
|
||||
$"No later purged archive provably covers it: {reason}"));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!dryRun)
|
||||
{
|
||||
DeleteArchiveFiles(stale);
|
||||
}
|
||||
|
||||
deleted.Add(stale.Name);
|
||||
items.Add(new AuditArchiveCleanupItem(stale.Name, stale.Partition, AuditArchiveCleanupAction.DeletedSuperseded,
|
||||
$"Superseded by {coveredBy.Name}, which is purged, verifies and holds every row of this archive."));
|
||||
}
|
||||
}
|
||||
|
||||
if (_options.RetentionDays > 0)
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
foreach (var manifest in manifests.Where(m => !deleted.Contains(m.Name)))
|
||||
{
|
||||
var monthEnd = AuditChainHasher.MonthOf(manifest.PartitionBoundaryUtc).AddMonths(1);
|
||||
if (monthEnd.AddDays(_options.RetentionDays) > now)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (manifest.Status != AuditArchiveStatus.Purged
|
||||
&& await PartitionHasRowsAsync(manifest.PartitionBoundaryUtc, ct).ConfigureAwait(false))
|
||||
{
|
||||
continue; // the partition was never switched out; this archive is not a sole copy, nor expired data gone
|
||||
}
|
||||
|
||||
if (!dryRun)
|
||||
{
|
||||
DeleteArchiveFiles(manifest);
|
||||
}
|
||||
|
||||
deleted.Add(manifest.Name);
|
||||
items.Add(new AuditArchiveCleanupItem(manifest.Name, manifest.Partition, AuditArchiveCleanupAction.DeletedExpired,
|
||||
$"{manifest.Partition} ended {monthEnd:yyyy-MM-dd}, more than {_options.RetentionDays} days ago (AuditLog:Archive:RetentionDays)."));
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var item in items.Where(i => i.Action != AuditArchiveCleanupAction.KeptStale))
|
||||
{
|
||||
_logger.LogInformation("AuditLog archive cleanup{DryRun}: {Action} {Archive} — {Detail}",
|
||||
dryRun ? " (dry run)" : "", item.Action, item.Name, item.Detail);
|
||||
}
|
||||
|
||||
return new AuditArchiveCleanupResult(dryRun, _options.RetentionDays, items);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Proves <paramref name="later"/> holds every row of <paramref name="stale"/>; returns null
|
||||
/// when it does, else why not.
|
||||
/// </summary>
|
||||
private async Task<string?> ProveCoveredAsync(AuditArchiveManifest stale, AuditArchiveManifest later, CancellationToken ct)
|
||||
{
|
||||
// Same chain generation and at least as far, for every chain month the stale archive has.
|
||||
var laterChains = later.Chains.ToDictionary(c => c.Partition, StringComparer.Ordinal);
|
||||
var targets = new Dictionary<(DateTime Month, long Seq), string>();
|
||||
foreach (var chain in stale.Chains)
|
||||
{
|
||||
if (!laterChains.TryGetValue(chain.Partition, out var next))
|
||||
{
|
||||
return $"{later.Name} has no chain for {chain.Partition}.";
|
||||
}
|
||||
|
||||
if (next.PriorPurgedThroughSeq != chain.PriorPurgedThroughSeq || next.LastSeq < chain.LastSeq)
|
||||
{
|
||||
return $"{later.Name}'s chain for {chain.Partition} does not extend this archive's (prior anchor or end differs).";
|
||||
}
|
||||
|
||||
if (chain.LastSeq > (chain.PriorPurgedThroughSeq ?? 0))
|
||||
{
|
||||
targets[(AuditArchiveParquet.ParseMonth(chain.Partition), chain.LastSeq)] = chain.LastHash.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
if (stale.UnchainedRowCount > MaxUnchainedRowsCompared)
|
||||
{
|
||||
return $"{stale.UnchainedRowCount} unchained rows exceed the {MaxUnchainedRowsCompared} compared by EventId.";
|
||||
}
|
||||
|
||||
// The later archive must itself be intact and tied to the database.
|
||||
var verification = await VerifyAsync(later.Name, ct).ConfigureAwait(false);
|
||||
if (!verification.Verified)
|
||||
{
|
||||
return $"{later.Name} does not verify ({verification.FileProblem ?? "chain or anchor problem"}).";
|
||||
}
|
||||
|
||||
var directory = ArchiveDirectory(stale);
|
||||
var unchained = new HashSet<Guid>();
|
||||
await using (var input = File.OpenRead(Path.Combine(directory, stale.Name + ".parquet")))
|
||||
{
|
||||
await foreach (var row in AuditArchiveParquet.ReadRowsAsync(input, ct).ConfigureAwait(false))
|
||||
{
|
||||
if (row.ChainSeq is null)
|
||||
{
|
||||
unchained.Add(row.EventId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var laterDirectory = ArchiveDirectory(later);
|
||||
await using (var input = File.OpenRead(Path.Combine(laterDirectory, later.Name + ".parquet")))
|
||||
{
|
||||
await foreach (var row in AuditArchiveParquet.ReadRowsAsync(input, ct).ConfigureAwait(false))
|
||||
{
|
||||
unchained.Remove(row.EventId);
|
||||
if (row.ChainSeq is { } seq
|
||||
&& targets.TryGetValue((AuditChainHasher.MonthOf(row.OccurredAtUtc), seq), out var expected))
|
||||
{
|
||||
if (row.RowHash is null || !string.Equals(Convert.ToHexStringLower(row.RowHash), expected, StringComparison.Ordinal))
|
||||
{
|
||||
return $"{later.Name} carries a different hash at the stale archive's chain end.";
|
||||
}
|
||||
|
||||
targets.Remove((AuditChainHasher.MonthOf(row.OccurredAtUtc), seq));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await using (var input = File.OpenRead(Path.Combine(laterDirectory, later.Name + ".tombstones.parquet")))
|
||||
{
|
||||
await foreach (var tombstone in AuditArchiveParquet.ReadTombstonesAsync(input, ct).ConfigureAwait(false))
|
||||
{
|
||||
unchained.Remove(tombstone.EventId);
|
||||
if (targets.TryGetValue((tombstone.ChainMonth, tombstone.ChainSeq), out var expected))
|
||||
{
|
||||
if (!string.Equals(Convert.ToHexStringLower(tombstone.RowHash), expected, StringComparison.Ordinal))
|
||||
{
|
||||
return $"{later.Name} carries a different hash at the stale archive's chain end.";
|
||||
}
|
||||
|
||||
targets.Remove((tombstone.ChainMonth, tombstone.ChainSeq));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (targets.Count > 0)
|
||||
{
|
||||
return $"{later.Name} does not contain the stale archive's chain end.";
|
||||
}
|
||||
|
||||
return unchained.Count == 0
|
||||
? null
|
||||
: $"{unchained.Count} unchained row(s) of this archive are not in {later.Name} (e.g. removed by a per-channel purge, which leaves no tombstone for an unchained row).";
|
||||
}
|
||||
|
||||
private async Task<bool> PartitionHasRowsAsync(DateTime boundary, CancellationToken ct)
|
||||
{
|
||||
var conn = _context.Database.GetDbConnection();
|
||||
var openedHere = await AuditLogHashChain.OpenAsync(conn, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT CASE WHEN EXISTS (SELECT 1 FROM dbo.AuditLog " +
|
||||
"WHERE $partition.pf_AuditLog_Month(OccurredAtUtc) = $partition.pf_AuditLog_Month(@boundary)) THEN 1 ELSE 0 END;";
|
||||
cmd.Parameters.Add(new SqlParameter("@boundary", SqlDbType.DateTime2) { Scale = 7, Value = boundary });
|
||||
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) == 1;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (openedHere)
|
||||
{
|
||||
await conn.CloseAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string ArchiveDirectory(AuditArchiveManifest manifest)
|
||||
{
|
||||
if (!IsValidName(manifest.Name))
|
||||
{
|
||||
throw new InvalidOperationException($"'{manifest.Name}' is not an archive name.");
|
||||
}
|
||||
|
||||
return Path.Combine(_options.Directory!, manifest.Name.Substring("auditlog-".Length, 4));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes exactly the archive's three files, manifest first. File names are derived from
|
||||
/// the validated archive name, never taken from the manifest.
|
||||
/// </summary>
|
||||
private void DeleteArchiveFiles(AuditArchiveManifest manifest)
|
||||
{
|
||||
var directory = Path.GetFullPath(ArchiveDirectory(manifest));
|
||||
var root = Path.GetFullPath(_options.Directory!);
|
||||
if (!directory.StartsWith(root, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Archive {manifest.Name} resolves outside {root}.");
|
||||
}
|
||||
|
||||
foreach (var file in new[] { ".manifest.json", ".parquet", ".tombstones.parquet" })
|
||||
{
|
||||
var path = Path.Combine(directory, manifest.Name + file);
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,6 +398,8 @@ public class ManagementActor : ReceiveActor
|
||||
// AuditLog Parquet archives (ScadaBridge#42): audit reads with verify-chain's gate —
|
||||
// OperationalAudit roles, site-scoped callers refused in the handler.
|
||||
ListAuditArchivesCommand => OperationalAuditReaders,
|
||||
// Archive cleanup deletes archive files (ScadaBridge#109): Administrator only.
|
||||
CleanupAuditArchivesCommand => AdminOnly,
|
||||
VerifyAuditArchiveCommand => OperationalAuditReaders,
|
||||
|
||||
// Read-only queries -- any authenticated user
|
||||
@@ -574,6 +576,7 @@ public class ManagementActor : ReceiveActor
|
||||
VerifyAuditChainHeadsCommand cmd => await HandleVerifyAuditChainHeads(sp, cmd, user),
|
||||
ListAuditArchivesCommand => await HandleListAuditArchives(sp, user),
|
||||
VerifyAuditArchiveCommand cmd => await HandleVerifyAuditArchive(sp, cmd, user),
|
||||
CleanupAuditArchivesCommand cmd => await HandleCleanupAuditArchives(sp, cmd, user),
|
||||
|
||||
// Health
|
||||
GetHealthSummaryCommand => HandleGetHealthSummary(sp),
|
||||
@@ -3340,6 +3343,20 @@ public class ManagementActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the archive cleanup now (ScadaBridge#109), on the maintenance-connection archive when
|
||||
/// registered. Dry run reports only.
|
||||
/// </summary>
|
||||
private static async Task<object?> HandleCleanupAuditArchives(
|
||||
IServiceProvider sp, CleanupAuditArchivesCommand cmd, AuthenticatedUser user)
|
||||
{
|
||||
RequireAuditArchive(sp, user);
|
||||
var archive = sp.GetKeyedService<IAuditLogArchive>(
|
||||
ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.AuditMaintenanceContext.ServiceKey)
|
||||
?? sp.GetRequiredService<IAuditLogArchive>();
|
||||
return await archive.CleanupAsync(cmd.DryRun);
|
||||
}
|
||||
|
||||
private static IAuditLogArchive RequireAuditArchive(IServiceProvider sp, AuthenticatedUser user)
|
||||
{
|
||||
if (user.PermittedSiteIds.Length > 0
|
||||
|
||||
@@ -69,6 +69,8 @@ public static class ManagementEndpoints
|
||||
// AuditLog archive verification (ScadaBridge#42) hashes and re-chains a whole month's
|
||||
// Parquet archive.
|
||||
typeof(VerifyAuditArchiveCommand),
|
||||
// Archive cleanup proves stale archives redundant by reading Parquet files (ScadaBridge#109).
|
||||
typeof(CleanupAuditArchivesCommand),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user