dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 17 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(audit): Parquet cold archive of expired AuditLog partitions before the purge (ScadaBridge#42)

Browse Source 
Audit roadmap T2 (alog.md §15.2): with AuditLog:Archive:Enabled (default off)
the purge actor writes each expired monthly partition to Parquet under an
absolute AuditLog:Archive:Directory before switching it out, so audit rows
outlive RetentionDays.

- Parquet.Net 6.1.0 (fully managed; NuGetAudit clean) in central package
  management.
- AuditArchiveParquet: data file = the ten stored columns, OccurredAtUtc as a
  UTC microsecond timestamp plus exact OccurredAtUtcTicks, and ChainSeq/RowHash;
  tombstone file = the month's per-channel purge tombstones. Buffered row
  groups; the files are plain Parquet (DuckDB/pyarrow read them).
- AuditLogPartitionArchive (IAuditLogArchive): reads the partition's rows,
  chain heads and tombstones in one transaction under the chain lock SHARED,
  writes {Directory}/{YYYY}/auditlog-{YYYY-MM}-{stamp}Z.parquet,
  .tombstones.parquet and a .manifest.json (counts, SHA-256 per file, each
  month's head and prior anchor) via tmp + rename, manifest last. Archives are
  never overwritten.
- Archive before purge: SwitchOutPartitionAsync gains an internal guarded
  overload. Under the EXCLUSIVE chain lock it re-counts the partition and
  re-reads each month's LastSeq and THROWs 51002/51003 (nothing purged) unless
  both still equal the archive's, so the switch's checkpoint anchor is exactly
  the archive's chain end. The manifest is then marked Purged.
- Fail closed: a partition that cannot be archived is not purged; the failure
  takes the existing AuditLogPurgeFailedEvent / PurgeFailures path. Success
  publishes AuditLogArchivedEvent.
- Verify: file hashes and counts against the manifest, the chain recomputed
  from the archived rows and tombstones (AuditChainVerifier), and the chain end
  against the database anchor (Matches / Superseded / NotPurged / Mismatch).
- AuditLogArchiveOptionsValidator: Directory required when enabled and must be
  absolute; RowGroupSize, timeouts bounded.

No migration: the purger role already reads AuditLog, the head and tombstones.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 17:00:49 -05:00
parent 31cb2c79e3
commit 7be4f19861
18 changed files with 2232 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogPurgeActorTests.cs
+104
View File
@@ -4,6 +4,7 @@ using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
using ZB.MOM.WW.Audit;
@@ -688,4 +689,107 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
StopPurgeActor(actor);
}
// ---------------------------------------------------------------------
// Parquet cold archive (ScadaBridge#42)
// ---------------------------------------------------------------------
private IActorRef CreateArchivingActor(RecordingRepo repo, IAuditLogArchive archive, bool enabled)
{
var services = new ServiceCollection();
services.AddScoped<IAuditLogRepository>(_ => repo);
services.AddScoped(_ => archive);
services.AddSingleton(Options.Create(new ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.AuditLogArchiveOptions
{
Enabled = enabled,
Directory = "/archive",
}));
var sp = services.BuildServiceProvider();
return Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
sp,
Options.Create(FastTickOptions(TimeSpan.FromHours(1))),
Options.Create(new AuditLogOptions()),
NullLogger<AuditLogPurgeActor>.Instance)));
}
private static AuditArchiveManifest ManifestFor(DateTime boundary) => new(
1, $"auditlog-{boundary:yyyy-MM}-20261109T120000000Z", boundary, boundary.ToString("yyyy-MM"),
DateTime.UtcNow, AuditArchiveStatus.Archived, null, 7, 7, 0, null, null,
"d.parquet", "00", 99, "t.parquet", "00", 0, []);
/// <summary>
/// With archiving enabled every expired partition is archived and then purged by the
/// archive's guarded purge — never by the plain, unguarded switch.
/// </summary>
[Fact]
public void ArchiveEnabled_ArchivesEachPartition_ThenPurgesThroughTheArchive()
{
var boundary = new DateTime(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc);
var repo = new RecordingRepo { Boundaries = [boundary] };
var archive = Substitute.For<IAuditLogArchive>();
var manifest = ManifestFor(boundary);
archive.ArchivePartitionAsync(boundary, Arg.Any<CancellationToken>()).Returns(manifest);
archive.PurgeArchivedPartitionAsync(manifest, Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
.Returns(7L);
var archived = CreateTestProbe();
Sys.EventStream.Subscribe(archived.Ref, typeof(AuditLogArchivedEvent));
var purged = SubscribePurged();
var actor = CreateArchivingActor(repo, archive, enabled: true);
actor.Tell(AuditLogPurgeActor.PurgeTick.Instance);
var a = archived.ExpectMsg<AuditLogArchivedEvent>(TimeSpan.FromSeconds(5));
Assert.Equal(manifest.Name, a.ArchiveName);
Assert.Equal(7, purged.ExpectMsg<AuditLogPurgedEvent>(TimeSpan.FromSeconds(5)).RowsDeleted);
Assert.Empty(repo.SwitchedBoundaries);
archive.Received(1).PurgeArchivedPartitionAsync(manifest, Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>());
StopPurgeActor(actor);
}
/// <summary>
/// Fail closed: a partition that could not be archived is not purged at all, and the
/// failure surfaces as a purge failure.
/// </summary>
[Fact]
public void ArchiveFailure_LeavesThePartitionInPlace_AndPublishesAFailure()
{
var boundary = new DateTime(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc);
var repo = new RecordingRepo { Boundaries = [boundary] };
var archive = Substitute.For<IAuditLogArchive>();
archive.ArchivePartitionAsync(boundary, Arg.Any<CancellationToken>())
.Returns<Task<AuditArchiveManifest>>(_ => throw new IOException("archive volume full"));
var failed = CreateTestProbe();
Sys.EventStream.Subscribe(failed.Ref, typeof(AuditLogPurgeFailedEvent));
var actor = CreateArchivingActor(repo, archive, enabled: true);
actor.Tell(AuditLogPurgeActor.PurgeTick.Instance);
var evt = failed.ExpectMsg<AuditLogPurgeFailedEvent>(TimeSpan.FromSeconds(5));
Assert.Equal(boundary, evt.MonthBoundary);
Assert.Contains("archive volume full", evt.Error);
Assert.Empty(repo.SwitchedBoundaries);
archive.DidNotReceiveWithAnyArgs().PurgeArchivedPartitionAsync(default!, default, default);
StopPurgeActor(actor);
}
/// <summary>With archiving configured but disabled, the purge is the plain switch and the archive is untouched.</summary>
[Fact]
public void ArchiveDisabled_UsesThePlainSwitch()
{
var boundary = new DateTime(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc);
var repo = new RecordingRepo { Boundaries = [boundary] };
var archive = Substitute.For<IAuditLogArchive>();
var purged = SubscribePurged();
var actor = CreateArchivingActor(repo, archive, enabled: false);
actor.Tell(AuditLogPurgeActor.PurgeTick.Instance);
purged.ExpectMsg<AuditLogPurgedEvent>(TimeSpan.FromSeconds(5));
Assert.Equal([boundary], repo.SwitchedBoundaries);
archive.DidNotReceiveWithAnyArgs().ArchivePartitionAsync(default, default);
StopPurgeActor(actor);
}
}