feat(audit): off-box chain-head export, verify-heads, maintenance login (ScadaBridge#107)
Item 1 (owner decision: no signing). AuditLog:HashChain:HeadExport (default
off): both centrals periodically read every month's chain head (LastSeq,
LastHash, purge anchor, chained rows, tombstones; one snapshot under the
chain lock SHARED) and append the changed heads to a sink the DB login cannot
write - File (append-only JSONL per node per day, local or UNC) or Syslog
(RFC 5424 over UDP / TCP octet counting). `audit verify-heads --path` reads
those records back (file or syslog-collector lines) and the new
VerifyAuditChainHeads command checks each against the database: Matches /
Extended / Purged, or Mismatch / Regressed / MissingHead / MissingLink /
CountMismatch (exit 3). AuditChainHeadStoreTests forges a chain with the
production hasher: verify-chain says Intact, verify-heads says Mismatch.
Item 3. AuditLog:MaintenanceConnectionString (${secret:}-capable, unset =
today's single login) runs the chain job, chain verify / head export, the
partition switch-out + archive and the per-channel purge on
AuditMaintenanceContext (keyed audit-maintenance repository/archive, purge
actor prefers them). AuditMaintenanceLoginTests runs the hardened setup on
SQL Server: writer (datareader/datawriter/audit_writer) ingests and is
refused chaining and deleting; the chainer+purger login does the full cycle.
Two-login grants documented in Component-AuditLog.md.
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -1,4 +1,5 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
@@ -33,6 +34,12 @@ public sealed class AuditLogHashChainOptions
|
||||
|
||||
/// <summary>Upper bound on batches per tick so a backlog yields between ticks (default 100).</summary>
|
||||
public int MaxBatchesPerTick { get; set; } = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Off-box export of the per-month chain heads (<c>AuditLog:HashChain:HeadExport</c>,
|
||||
/// ScadaBridge#107 item 1) — the chain's external anchor. Default off.
|
||||
/// </summary>
|
||||
public AuditChainHeadExportOptions HeadExport { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>Validates <see cref="AuditLogHashChainOptions"/> at host startup.</summary>
|
||||
@@ -47,5 +54,33 @@ public sealed class AuditLogHashChainOptionsValidator : OptionsValidatorBase<Aud
|
||||
$"AuditLog:HashChain:{nameof(AuditLogHashChainOptions.BatchSize)} ({options.BatchSize}) must be between 1 and 10000.");
|
||||
builder.RequireThat(options.MaxBatchesPerTick > 0,
|
||||
$"AuditLog:HashChain:{nameof(AuditLogHashChainOptions.MaxBatchesPerTick)} ({options.MaxBatchesPerTick}) must be > 0.");
|
||||
|
||||
var export = options.HeadExport ?? new AuditChainHeadExportOptions();
|
||||
if (!export.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string section = "AuditLog:HashChain:HeadExport";
|
||||
builder.RequireThat(export.IntervalMinutes > 0,
|
||||
$"{section}:IntervalMinutes ({export.IntervalMinutes}) must be > 0.");
|
||||
if (export.Sink == AuditChainHeadSinkKind.File)
|
||||
{
|
||||
var dir = export.File?.Directory;
|
||||
builder.RequireThat(!string.IsNullOrWhiteSpace(dir) && Path.IsPathFullyQualified(dir),
|
||||
$"{section}:File:Directory ('{dir}') must be an absolute path (local or UNC) when Sink=File.");
|
||||
}
|
||||
else
|
||||
{
|
||||
var syslog = export.Syslog ?? new AuditChainHeadSyslogSinkOptions();
|
||||
builder.RequireThat(!string.IsNullOrWhiteSpace(syslog.Host),
|
||||
$"{section}:Syslog:Host is required when Sink=Syslog.");
|
||||
builder.RequireThat(syslog.Port is > 0 and <= 65535,
|
||||
$"{section}:Syslog:Port ({syslog.Port}) must be between 1 and 65535.");
|
||||
builder.RequireThat(syslog.Facility is >= 0 and <= 23,
|
||||
$"{section}:Syslog:Facility ({syslog.Facility}) must be between 0 and 23.");
|
||||
builder.RequireThat(syslog.TimeoutSeconds > 0,
|
||||
$"{section}:Syslog:TimeoutSeconds ({syslog.TimeoutSeconds}) must be > 0.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>DI scopes.</b> <see cref="IAuditLogRepository"/> is a scoped EF Core
|
||||
/// service registered by <c>AddConfigurationDatabase</c>. The singleton
|
||||
/// service registered by <c>AddConfigurationDatabase</c> — the
|
||||
/// <see cref="AuditMaintenanceContext.ServiceKey"/>-keyed one, on the AuditLog maintenance
|
||||
/// connection (ScadaBridge#107 item 3), when registered. The singleton
|
||||
/// opens one DI scope per tick and reuses the same repository across every
|
||||
/// boundary in that tick — mirrors the
|
||||
/// <see cref="SiteAuditReconciliationActor"/> pattern.
|
||||
@@ -149,7 +151,11 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
IAuditLogRepository repository;
|
||||
try
|
||||
{
|
||||
repository = scope.ServiceProvider.GetRequiredService<IAuditLogRepository>();
|
||||
// The maintenance-connection repository when the host registered one
|
||||
// (AddConfigurationDatabase, ScadaBridge#107 item 3 — on the separate maintenance
|
||||
// login if AuditLog:MaintenanceConnectionString is set), else the plain one.
|
||||
repository = scope.ServiceProvider.GetKeyedService<IAuditLogRepository>(AuditMaintenanceContext.ServiceKey)
|
||||
?? scope.ServiceProvider.GetRequiredService<IAuditLogRepository>();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -202,7 +208,8 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
{
|
||||
try
|
||||
{
|
||||
archive = scope.ServiceProvider.GetRequiredService<IAuditLogArchive>();
|
||||
archive = scope.ServiceProvider.GetKeyedService<IAuditLogArchive>(AuditMaintenanceContext.ServiceKey)
|
||||
?? scope.ServiceProvider.GetRequiredService<IAuditLogArchive>();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
/// <summary>Where exported chain heads go.</summary>
|
||||
public enum AuditChainHeadSinkKind
|
||||
{
|
||||
/// <summary>Append-only JSON-lines files in a directory (local or UNC).</summary>
|
||||
File,
|
||||
|
||||
/// <summary>RFC 5424 syslog over UDP or TCP.</summary>
|
||||
Syslog,
|
||||
}
|
||||
|
||||
/// <summary>Syslog transport.</summary>
|
||||
public enum AuditSyslogProtocol
|
||||
{
|
||||
/// <summary>One datagram per record (RFC 5426).</summary>
|
||||
Udp,
|
||||
|
||||
/// <summary>Octet-counted frames on one TCP connection per export (RFC 6587).</summary>
|
||||
Tcp,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Off-box export of the AuditLog hash-chain heads (section <c>AuditLog:HashChain:HeadExport</c>,
|
||||
/// ScadaBridge#107 item 1). <b>Default OFF.</b> Every <see cref="IntervalMinutes"/> both central
|
||||
/// nodes read each month's chain head (month, last sequence + hash, partition-purge anchor,
|
||||
/// chained-row and tombstone counts) and append the heads that changed since their previous
|
||||
/// export to a sink the database login cannot write. <c>audit verify-heads</c> later compares
|
||||
/// those records with the database: a chain someone rewrote and re-hashed inside SQL Server no
|
||||
/// longer carries the hash that was exported.
|
||||
/// </summary>
|
||||
public sealed class AuditChainHeadExportOptions
|
||||
{
|
||||
/// <summary>Whether the export runs (default false).</summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>Minutes between exports (default 60).</summary>
|
||||
public int IntervalMinutes { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Re-export unchanged heads every tick too (default false: only heads whose sequence,
|
||||
/// anchor or counts moved since this node's previous export, plus every head on the first
|
||||
/// tick after start).
|
||||
/// </summary>
|
||||
public bool ExportUnchanged { get; set; }
|
||||
|
||||
/// <summary>Which sink (default <see cref="AuditChainHeadSinkKind.File"/>).</summary>
|
||||
public AuditChainHeadSinkKind Sink { get; set; } = AuditChainHeadSinkKind.File;
|
||||
|
||||
/// <summary>File sink settings.</summary>
|
||||
public AuditChainHeadFileSinkOptions File { get; set; } = new();
|
||||
|
||||
/// <summary>Syslog sink settings.</summary>
|
||||
public AuditChainHeadSyslogSinkOptions Syslog { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>File sink: <c>{Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl</c>, opened for append only.</summary>
|
||||
public sealed class AuditChainHeadFileSinkOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Absolute directory (local path or UNC share). Give the service account create + append
|
||||
/// rights only; the SQL Server login has no access to it by construction.
|
||||
/// </summary>
|
||||
public string? Directory { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Syslog sink (RFC 5424).</summary>
|
||||
public sealed class AuditChainHeadSyslogSinkOptions
|
||||
{
|
||||
/// <summary>Collector host name or address.</summary>
|
||||
public string? Host { get; set; }
|
||||
|
||||
/// <summary>Collector port (default 514).</summary>
|
||||
public int Port { get; set; } = 514;
|
||||
|
||||
/// <summary>Transport (default UDP).</summary>
|
||||
public AuditSyslogProtocol Protocol { get; set; } = AuditSyslogProtocol.Udp;
|
||||
|
||||
/// <summary>Syslog facility 0–23 (default 13, "log audit").</summary>
|
||||
public int Facility { get; set; } = 13;
|
||||
|
||||
/// <summary>APP-NAME field (default <c>scadabridge</c>).</summary>
|
||||
public string AppName { get; set; } = "scadabridge";
|
||||
|
||||
/// <summary>Send/connect timeout in seconds (default 10).</summary>
|
||||
public int TimeoutSeconds { get; set; } = 10;
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
/// <summary>
|
||||
/// Central hosted service that exports the AuditLog hash-chain heads off-box
|
||||
/// (<c>AuditLog:HashChain:HeadExport</c>, ScadaBridge#107 item 1). Default off.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Runs on both central nodes (each record names its node). A tick reads every month's head in
|
||||
/// one consistent snapshot (<see cref="IAuditChainHeadStore.ReadHeadsAsync"/>, chain lock
|
||||
/// SHARED) and writes the heads that changed since this node's last successful export — all of
|
||||
/// them on the first tick after start — to the configured <see cref="IAuditChainHeadSink"/>.
|
||||
/// A failed write is logged and retried on the next tick with the same heads (the in-memory
|
||||
/// "last exported" state only advances on success). No signing: the protection is that the
|
||||
/// sink sits outside the database login's reach.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class AuditChainHeadExportService : IHostedService, IDisposable
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly AuditChainHeadExportOptions _options;
|
||||
private readonly IAuditChainHeadSink? _sink;
|
||||
private readonly string? _node;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly ILogger<AuditChainHeadExportService> _logger;
|
||||
private readonly Dictionary<DateTime, (long Seq, long? Purged, long Links)> _lastExported = new();
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _loop;
|
||||
|
||||
/// <summary>Initializes the service.</summary>
|
||||
/// <param name="scopeFactory">Scope factory for the scoped head store.</param>
|
||||
/// <param name="options">Hash-chain options (the <c>HeadExport</c> sub-section is used).</param>
|
||||
/// <param name="sink">The sink (null when the export is disabled).</param>
|
||||
/// <param name="nodeName">This node's name for the records.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
/// <param name="time">Clock.</param>
|
||||
public AuditChainHeadExportService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptions<AuditLogHashChainOptions> options,
|
||||
IAuditChainHeadSink? sink,
|
||||
string? nodeName,
|
||||
ILogger<AuditChainHeadExportService> logger,
|
||||
TimeProvider? time = null)
|
||||
{
|
||||
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.HeadExport;
|
||||
_sink = sink;
|
||||
_node = nodeName;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_time = time ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_options.Enabled || _sink is null)
|
||||
{
|
||||
_logger.LogInformation("AuditLog chain-head export disabled (AuditLog:HashChain:HeadExport:Enabled=false).");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
_logger.LogInformation("AuditLog chain-head export every {Minutes} min to {Sink}.", _options.IntervalMinutes, _sink.Describe);
|
||||
var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
_cts = cts;
|
||||
var token = cts.Token;
|
||||
_loop = Task.Run(() => RunLoopAsync(token), CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task RunLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
await TickAsync(ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMinutes(_options.IntervalMinutes), ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs one export tick.</summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The number of records written (0 on failure or nothing changed).</returns>
|
||||
internal async Task<int> TickAsync(CancellationToken ct)
|
||||
{
|
||||
if (_sink is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
IReadOnlyList<AuditChainHeadSnapshot> heads;
|
||||
await using (var scope = _scopeFactory.CreateAsyncScope())
|
||||
{
|
||||
heads = await scope.ServiceProvider.GetRequiredService<IAuditChainHeadStore>()
|
||||
.ReadHeadsAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var changed = heads
|
||||
.Where(h => _options.ExportUnchanged
|
||||
|| !_lastExported.TryGetValue(h.Month, out var last)
|
||||
|| last != (h.LastSeq, h.PurgedThroughSeq, h.ChainedRows + h.Tombstones))
|
||||
.ToList();
|
||||
if (changed.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var records = changed.Select(h => ToRecord(h, now, _node)).ToList();
|
||||
await _sink.WriteAsync(records, ct).ConfigureAwait(false);
|
||||
foreach (var h in changed)
|
||||
{
|
||||
_lastExported[h.Month] = (h.LastSeq, h.PurgedThroughSeq, h.ChainedRows + h.Tombstones);
|
||||
}
|
||||
|
||||
_logger.LogDebug("Exported {Count} AuditLog chain head(s) to {Sink}.", records.Count, _sink.Describe);
|
||||
return records.Count;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Contained: the next tick retries the same heads (nothing was marked exported).
|
||||
_logger.LogError(ex, "AuditLog chain-head export to {Sink} failed; retrying next tick.", _sink.Describe);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Maps a database head to its exported record.</summary>
|
||||
/// <param name="head">The head.</param>
|
||||
/// <param name="exportedAtUtc">Export time.</param>
|
||||
/// <param name="node">Exporting node.</param>
|
||||
/// <returns>The record.</returns>
|
||||
public static AuditChainHeadRecord ToRecord(AuditChainHeadSnapshot head, DateTime exportedAtUtc, string? node)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(head);
|
||||
return new AuditChainHeadRecord(
|
||||
AuditChainHeadRecord.CurrentVersion,
|
||||
AuditChainHasher.PartitionLabel(head.Month),
|
||||
head.LastSeq,
|
||||
AuditChainHeadComparer.ToHex(head.LastHash),
|
||||
head.PurgedThroughSeq,
|
||||
head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash),
|
||||
head.ChainedRows,
|
||||
head.Tombstones,
|
||||
head.UpdatedAtUtc,
|
||||
exportedAtUtc,
|
||||
node);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
_cts?.Cancel();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Dispose already cancelled the loop.
|
||||
}
|
||||
|
||||
return _loop ?? Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
_cts?.Cancel();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Idempotent.
|
||||
}
|
||||
|
||||
_cts?.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
/// <summary>The one JSON shape every sink writes (and <c>audit verify-heads</c> reads).</summary>
|
||||
public static class AuditChainHeadJson
|
||||
{
|
||||
/// <summary>camelCase, compact, nulls kept so every record has every field.</summary>
|
||||
public static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
|
||||
};
|
||||
|
||||
/// <summary>Serialises one record to a single line.</summary>
|
||||
/// <param name="record">The record.</param>
|
||||
/// <returns>Compact JSON.</returns>
|
||||
public static string Serialize(AuditChainHeadRecord record) => JsonSerializer.Serialize(record, Options);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using System.Text;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
/// <summary>
|
||||
/// Appends exported heads as JSON lines to <c>{Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The file is only ever opened with <see cref="FileMode.Append"/> — never truncated, rewritten
|
||||
/// or deleted by ScadaBridge — so the share can grant the service account create + append data
|
||||
/// only. One file per node per UTC day keeps the two central nodes from interleaving writes in
|
||||
/// one file. Each batch is flushed to disk before the call returns.
|
||||
/// </remarks>
|
||||
public sealed class FileAuditChainHeadSink : IAuditChainHeadSink
|
||||
{
|
||||
private readonly string _directory;
|
||||
private readonly string _node;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
/// <summary>Creates the sink.</summary>
|
||||
/// <param name="directory">Absolute export directory.</param>
|
||||
/// <param name="node">This node's name (part of the file name).</param>
|
||||
/// <param name="time">Clock (file-per-day naming).</param>
|
||||
public FileAuditChainHeadSink(string directory, string node, TimeProvider? time = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(directory);
|
||||
if (!Path.IsPathFullyQualified(directory))
|
||||
{
|
||||
throw new ArgumentException($"Head-export directory '{directory}' must be absolute.", nameof(directory));
|
||||
}
|
||||
|
||||
_directory = directory;
|
||||
_node = SafeName(string.IsNullOrWhiteSpace(node) ? "node" : node);
|
||||
_time = time ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Describe => _directory;
|
||||
|
||||
/// <summary>The file a write at <paramref name="nowUtc"/> lands in.</summary>
|
||||
/// <param name="nowUtc">The write time.</param>
|
||||
/// <returns>The full path.</returns>
|
||||
public string PathFor(DateTime nowUtc) => Path.Combine(
|
||||
_directory, nowUtc.ToString("yyyy", System.Globalization.CultureInfo.InvariantCulture),
|
||||
$"audit-chain-heads-{nowUtc:yyyy-MM-dd}-{_node}.jsonl");
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task WriteAsync(IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(records);
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var path = PathFor(_time.GetUtcNow().UtcDateTime);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
|
||||
var text = new StringBuilder();
|
||||
foreach (var record in records)
|
||||
{
|
||||
text.Append(AuditChainHeadJson.Serialize(record)).Append('\n');
|
||||
}
|
||||
|
||||
var bytes = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false).GetBytes(text.ToString());
|
||||
await using var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read, 4096, useAsync: true);
|
||||
await stream.WriteAsync(bytes, ct).ConfigureAwait(false);
|
||||
await stream.FlushAsync(ct).ConfigureAwait(false);
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
|
||||
private static string SafeName(string value)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
var chars = value.Select(c => invalid.Contains(c) || c == ' ' ? '_' : c).ToArray();
|
||||
return new string(chars);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
/// <summary>An off-box destination for exported chain heads (ScadaBridge#107 item 1).</summary>
|
||||
public interface IAuditChainHeadSink
|
||||
{
|
||||
/// <summary>Short description for logs (e.g. the directory or <c>host:port/udp</c>).</summary>
|
||||
string Describe { get; }
|
||||
|
||||
/// <summary>Writes the records; throws when they could not all be delivered.</summary>
|
||||
/// <param name="records">The records, oldest month first.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that completes when the records are durably handed off.</returns>
|
||||
Task WriteAsync(IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using System.Globalization;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
||||
|
||||
/// <summary>
|
||||
/// Sends exported heads as RFC 5424 syslog messages — one message per head, the JSON record as
|
||||
/// MSG — over UDP (RFC 5426, one datagram each) or TCP (RFC 6587 octet-counting framing, one
|
||||
/// connection per export).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Message: <c><PRI>1 TIMESTAMP HOSTNAME APP-NAME PROCID chainhead - {json}</c>, severity
|
||||
/// Notice (5), PRI = facility × 8 + 5. No structured data (a private enterprise number would be
|
||||
/// needed); the collector keeps the JSON body, which <c>audit verify-heads</c> reads back from
|
||||
/// any line that contains it. UDP is fire-and-forget — use TCP where a lost datagram matters.
|
||||
/// </remarks>
|
||||
public sealed class SyslogAuditChainHeadSink : IAuditChainHeadSink
|
||||
{
|
||||
/// <summary>MSGID of every head message.</summary>
|
||||
public const string MessageId = "chainhead";
|
||||
|
||||
private const int SeverityNotice = 5;
|
||||
|
||||
private readonly AuditChainHeadSyslogSinkOptions _options;
|
||||
private readonly string _hostName;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
/// <summary>Creates the sink.</summary>
|
||||
/// <param name="options">Syslog settings.</param>
|
||||
/// <param name="hostName">HOSTNAME field (defaults to the machine name).</param>
|
||||
/// <param name="time">Clock for TIMESTAMP.</param>
|
||||
public SyslogAuditChainHeadSink(AuditChainHeadSyslogSinkOptions options, string? hostName = null, TimeProvider? time = null)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(options.Host);
|
||||
_hostName = Token(hostName ?? Environment.MachineName, 255);
|
||||
_time = time ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Describe => $"{_options.Host}:{_options.Port}/{_options.Protocol.ToString().ToLowerInvariant()}";
|
||||
|
||||
/// <summary>Formats one RFC 5424 message (no framing).</summary>
|
||||
/// <param name="record">The record.</param>
|
||||
/// <returns>The message text.</returns>
|
||||
public string Format(AuditChainHeadRecord record)
|
||||
{
|
||||
var pri = (_options.Facility * 8) + SeverityNotice;
|
||||
var timestamp = _time.GetUtcNow().UtcDateTime.ToString("yyyy-MM-dd'T'HH:mm:ss.fff'Z'", CultureInfo.InvariantCulture);
|
||||
return string.Create(CultureInfo.InvariantCulture,
|
||||
$"<{pri}>1 {timestamp} {_hostName} {Token(_options.AppName, 48)} {Environment.ProcessId} {MessageId} - {AuditChainHeadJson.Serialize(record)}");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task WriteAsync(IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(records);
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
timeout.CancelAfter(TimeSpan.FromSeconds(Math.Max(1, _options.TimeoutSeconds)));
|
||||
var token = timeout.Token;
|
||||
|
||||
if (_options.Protocol == AuditSyslogProtocol.Udp)
|
||||
{
|
||||
using var udp = new UdpClient();
|
||||
udp.Connect(_options.Host!, _options.Port);
|
||||
foreach (var record in records)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(Format(record));
|
||||
await udp.SendAsync(bytes, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
using var tcp = new TcpClient();
|
||||
await tcp.ConnectAsync(_options.Host!, _options.Port, token).ConfigureAwait(false);
|
||||
var stream = tcp.GetStream();
|
||||
foreach (var record in records)
|
||||
{
|
||||
var message = Encoding.UTF8.GetBytes(Format(record));
|
||||
var frame = Encoding.ASCII.GetBytes(message.Length.ToString(CultureInfo.InvariantCulture) + " ");
|
||||
await stream.WriteAsync(frame, token).ConfigureAwait(false);
|
||||
await stream.WriteAsync(message, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await stream.FlushAsync(token).ConfigureAwait(false);
|
||||
tcp.Client.Shutdown(SocketShutdown.Send);
|
||||
}
|
||||
|
||||
// RFC 5424 header fields are PRINTUSASCII (33-126), no spaces; "-" when empty.
|
||||
private static string Token(string? value, int max)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
return "-";
|
||||
}
|
||||
|
||||
var chars = value.Where(c => c is >= '!' and <= '~').Take(max).ToArray();
|
||||
return chars.Length == 0 ? "-" : new string(chars);
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,12 @@ public static class ServiceCollectionExtensions
|
||||
/// <summary>Configuration section bound to <see cref="AuditLogPartitionMaintenanceOptions"/>.</summary>
|
||||
public const string PartitionMaintenanceSectionName = "AuditLog:PartitionMaintenance";
|
||||
|
||||
/// <summary>
|
||||
/// Optional connection string for the AuditLog maintenance paths — the hash-chain job and both
|
||||
/// purge paths (ScadaBridge#107 item 3). Unset: they run on <c>ScadaBridge:Database:ConfigurationDb</c>.
|
||||
/// </summary>
|
||||
public const string MaintenanceConnectionStringKey = "AuditLog:MaintenanceConnectionString";
|
||||
|
||||
/// <summary>Configuration section bound to <see cref="AuditLogHashChainOptions"/> (ScadaBridge#41).</summary>
|
||||
public const string HashChainSectionName = "AuditLog:HashChain";
|
||||
|
||||
@@ -411,6 +417,26 @@ public static class ServiceCollectionExtensions
|
||||
config, HashChainSectionName);
|
||||
services.AddHostedService<AuditLogHashChainService>();
|
||||
|
||||
// Off-box chain-head export (ScadaBridge#107 item 1, AuditLog:HashChain:HeadExport,
|
||||
// default off). The sink is built from the bound options; null when disabled.
|
||||
services.AddHostedService(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<AuditLogHashChainOptions>>();
|
||||
var node = sp.GetService<INodeIdentityProvider>()?.NodeName;
|
||||
var export = options.Value.HeadExport;
|
||||
Central.HeadExport.IAuditChainHeadSink? sink = !export.Enabled ? null : export.Sink switch
|
||||
{
|
||||
Central.HeadExport.AuditChainHeadSinkKind.Syslog => new Central.HeadExport.SyslogAuditChainHeadSink(export.Syslog),
|
||||
_ => new Central.HeadExport.FileAuditChainHeadSink(export.File.Directory!, node ?? Environment.MachineName),
|
||||
};
|
||||
return new Central.HeadExport.AuditChainHeadExportService(
|
||||
sp.GetRequiredService<IServiceScopeFactory>(),
|
||||
options,
|
||||
sink,
|
||||
node,
|
||||
sp.GetRequiredService<ILogger<Central.HeadExport.AuditChainHeadExportService>>());
|
||||
});
|
||||
|
||||
// Parquet cold archive (ScadaBridge#42). Default OFF; when enabled the purge actor
|
||||
// archives each expired partition before switching it out. IAuditLogArchive is
|
||||
// registered by AddConfigurationDatabase (central only) and reads these options.
|
||||
|
||||
@@ -10,8 +10,10 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
||||
/// (<c>GET /api/audit/query</c>, <c>GET /api/audit/export</c>,
|
||||
/// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
|
||||
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
|
||||
/// and <c>archive list</c> / <c>archive verify</c>, which send <c>ListAuditArchives</c> /
|
||||
/// <c>VerifyAuditArchive</c> (Parquet cold archive, ScadaBridge#42).
|
||||
/// <c>verify-heads</c>, which checks off-box exported chain heads with
|
||||
/// <c>VerifyAuditChainHeads</c> (ScadaBridge#107), and <c>archive list</c> / <c>archive verify</c>,
|
||||
/// which send <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c> (Parquet cold archive,
|
||||
/// ScadaBridge#42).
|
||||
/// </summary>
|
||||
public static class AuditCommands
|
||||
{
|
||||
@@ -32,6 +34,7 @@ public static class AuditCommands
|
||||
command.Add(BuildExport(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildVerifyHeads(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildArchive(urlOption, formatOption, usernameOption, passwordOption));
|
||||
|
||||
@@ -323,6 +326,54 @@ public static class AuditCommands
|
||||
/// Builds the <c>audit archive</c> group (ScadaBridge#42): <c>list</c> and <c>verify</c>
|
||||
/// over the AuditLog Parquet cold archives on the central archive directory.
|
||||
/// </summary>
|
||||
private static Command BuildVerifyHeads(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
|
||||
{
|
||||
var pathOption = new Option<string>("--path")
|
||||
{
|
||||
Description = "Export file, or directory of *.jsonl/*.log/*.txt export files (file sink or syslog collector output)",
|
||||
Required = true,
|
||||
};
|
||||
var cmd = new Command("verify-heads")
|
||||
{
|
||||
Description = "Check AuditLog chain heads exported off-box against the database chain; exit 3 if any no longer matches",
|
||||
};
|
||||
cmd.Add(pathOption);
|
||||
cmd.SetAction(async (ParseResult result) =>
|
||||
{
|
||||
AuditVerifyHeadsHelpers.ReadResult read;
|
||||
try
|
||||
{
|
||||
read = AuditVerifyHeadsHelpers.ReadRecords(result.GetValue(pathOption)!);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
OutputFormatter.WriteError(ex.Message, "INVALID_ARGUMENT");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (read.Records.Count == 0)
|
||||
{
|
||||
OutputFormatter.WriteError(
|
||||
$"No chain-head records found in {read.Files} file(s) ({read.LinesSkipped} unparsable line(s)).",
|
||||
"INVALID_ARGUMENT");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (read.LinesSkipped > 0)
|
||||
{
|
||||
Console.Error.WriteLine($"warning: skipped {read.LinesSkipped} line(s) without a chain-head record.");
|
||||
}
|
||||
|
||||
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
|
||||
return await CommandHelpers.ExecuteCommandAsync(
|
||||
result, urlOption, formatOption, usernameOption, passwordOption,
|
||||
new VerifyAuditChainHeadsCommand(read.Records),
|
||||
timeout: AuditVerifyHeadsHelpers.RequestTimeout,
|
||||
onSuccess: json => AuditVerifyHeadsHelpers.Render(json, format, Console.Out));
|
||||
});
|
||||
return cmd;
|
||||
}
|
||||
|
||||
private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
|
||||
{
|
||||
var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" };
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Helpers for <c>audit verify-heads</c> (ScadaBridge#107 item 1): read AuditLog chain heads
|
||||
/// exported off-box (the <c>AuditLog:HashChain:HeadExport</c> JSON lines — straight from the
|
||||
/// file sink, or from a syslog collector's log), send them in a
|
||||
/// <c>VerifyAuditChainHeads</c> management command, and render the verdicts.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Exit codes: <c>0</c> every exported head is consistent with the database,
|
||||
/// <see cref="FailedExitCode"/> at least one is not (tamper evidence), <c>1</c> on a request
|
||||
/// or input error, <c>2</c> on an authorization failure.
|
||||
/// </remarks>
|
||||
public static class AuditVerifyHeadsHelpers
|
||||
{
|
||||
/// <summary>Process exit code when a head check failed.</summary>
|
||||
public const int FailedExitCode = 3;
|
||||
|
||||
/// <summary>Client deadline (the server bounds the call with the long-running Ask).</summary>
|
||||
public static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(6);
|
||||
|
||||
private static readonly JsonSerializerOptions ReadOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
/// <summary>The result of reading export files.</summary>
|
||||
/// <param name="Records">Distinct records (by month, sequence, hash), latest export kept.</param>
|
||||
/// <param name="LinesRead">Lines that held a head record.</param>
|
||||
/// <param name="LinesSkipped">Non-empty lines without a parsable record.</param>
|
||||
/// <param name="Files">Files read.</param>
|
||||
public sealed record ReadResult(IReadOnlyList<AuditChainHeadRecord> Records, int LinesRead, int LinesSkipped, int Files);
|
||||
|
||||
/// <summary>
|
||||
/// Reads every record from <paramref name="path"/> — one file, or every <c>*.jsonl</c> /
|
||||
/// <c>*.log</c> / <c>*.txt</c> file below a directory. Each line's JSON object is taken from
|
||||
/// its first <c>{</c> to its last <c>}</c>, so syslog-prefixed lines parse too.
|
||||
/// </summary>
|
||||
/// <param name="path">File or directory.</param>
|
||||
/// <returns>The records read.</returns>
|
||||
/// <exception cref="FileNotFoundException">The path does not exist.</exception>
|
||||
public static ReadResult ReadRecords(string path)
|
||||
{
|
||||
string[] files;
|
||||
if (File.Exists(path))
|
||||
{
|
||||
files = [path];
|
||||
}
|
||||
else if (Directory.Exists(path))
|
||||
{
|
||||
files = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)
|
||||
.Where(f => f.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase)
|
||||
|| f.EndsWith(".log", StringComparison.OrdinalIgnoreCase)
|
||||
|| f.EndsWith(".txt", StringComparison.OrdinalIgnoreCase))
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new FileNotFoundException($"'{path}' is neither a file nor a directory.", path);
|
||||
}
|
||||
|
||||
var records = new List<AuditChainHeadRecord>();
|
||||
int read = 0, skipped = 0;
|
||||
foreach (var file in files)
|
||||
{
|
||||
foreach (var line in File.ReadLines(file))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = TryParseLine(line);
|
||||
if (record is null)
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
read++;
|
||||
records.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
var distinct = records
|
||||
.GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash.ToLowerInvariant()))
|
||||
.Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First())
|
||||
.OrderBy(r => r.Month, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.LastSeq)
|
||||
.ToList();
|
||||
return new ReadResult(distinct, read, skipped, files.Length);
|
||||
}
|
||||
|
||||
/// <summary>Parses one export line; null when it holds no valid record.</summary>
|
||||
/// <param name="line">The line.</param>
|
||||
/// <returns>The record, or null.</returns>
|
||||
public static AuditChainHeadRecord? TryParseLine(string line)
|
||||
{
|
||||
var start = line.IndexOf('{');
|
||||
var end = line.LastIndexOf('}');
|
||||
if (start < 0 || end <= start)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var record = JsonSerializer.Deserialize<AuditChainHeadRecord>(line.AsSpan(start, end - start + 1), ReadOptions);
|
||||
return record is { V: >= 1, Month.Length: 7, LastHash.Length: > 0 } && record.LastSeq > 0 ? record : null;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Renders a <c>VerifyAuditChainHeads</c> result and maps it to the exit code.</summary>
|
||||
/// <param name="json">Success body (an <c>AuditChainHeadVerificationResult</c>, camelCase).</param>
|
||||
/// <param name="format">Resolved output format.</param>
|
||||
/// <param name="output">Writer for the report.</param>
|
||||
/// <returns><c>0</c> when verified, <see cref="FailedExitCode"/> otherwise.</returns>
|
||||
public static int Render(string json, string format, TextWriter output)
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
var verified = root.TryGetProperty("verified", out var v) && v.ValueKind == JsonValueKind.True;
|
||||
|
||||
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
output.WriteLine(json);
|
||||
output.Flush();
|
||||
return verified ? 0 : FailedExitCode;
|
||||
}
|
||||
|
||||
output.WriteLine($"Verified : {(verified ? "yes" : "NO")}");
|
||||
output.WriteLine($"Heads checked : {Num(root, "recordsChecked")}");
|
||||
if (root.TryGetProperty("checks", out var checks) && checks.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
output.WriteLine($"{"MONTH",-8} {"EXPORTED SEQ",12} {"DB SEQ",8} {"STATUS",-13} DETAIL");
|
||||
foreach (var c in checks.EnumerateArray())
|
||||
{
|
||||
output.WriteLine(string.Create(CultureInfo.InvariantCulture,
|
||||
$"{Str(c, "month"),-8} {Num(c, "exportedSeq"),12} {Num(c, "databaseSeq", "-"),8} {Str(c, "status"),-13} {Str(c, "detail")}"));
|
||||
}
|
||||
}
|
||||
|
||||
output.Flush();
|
||||
return verified ? 0 : FailedExitCode;
|
||||
}
|
||||
|
||||
private static string Str(JsonElement e, string name)
|
||||
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? "" : "";
|
||||
|
||||
private static string Num(JsonElement e, string name, string missing = "0")
|
||||
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number
|
||||
? v.GetInt64().ToString(CultureInfo.InvariantCulture)
|
||||
: missing;
|
||||
}
|
||||
@@ -2071,6 +2071,41 @@ end, `DuplicateSequence`, `IncompleteLink`). `--format table` prints a summary;
|
||||
The chain only exists once `AuditLog:HashChain:Enabled` is on for the central nodes
|
||||
(default off); see `Component-AuditLog.md`, Security & Tamper-Evidence.
|
||||
|
||||
#### `audit verify-heads`
|
||||
|
||||
Check AuditLog chain heads that were exported off-box (ScadaBridge#107) against the
|
||||
database chain.
|
||||
|
||||
```sh
|
||||
scadabridge --url <url> audit verify-heads --path <file-or-directory>
|
||||
```
|
||||
|
||||
| Option | Required | Default | Description |
|
||||
|--------|----------|---------|-------------|
|
||||
| `--path` | yes | — | An export file, or a directory read recursively for `*.jsonl` / `*.log` / `*.txt` |
|
||||
|
||||
The central nodes write one JSON record per month head (`month`, `lastSeq`, `lastHash`,
|
||||
purge anchor, chained-row and tombstone counts, `exportedAtUtc`, `node`) when
|
||||
`AuditLog:HashChain:HeadExport:Enabled` is on — to append-only files
|
||||
(`{Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl`) or to syslog (RFC 5424, the
|
||||
JSON record as the message). The CLI reads both forms (the JSON object is taken from each
|
||||
line's first `{` to its last `}`, so a syslog collector's prefixes do not matter), keeps one
|
||||
record per distinct (month, sequence, hash), and sends `VerifyAuditChainHeads`. Per record the
|
||||
server reports `Matches`, `Extended`, `Purged` (the link was removed by a partition purge —
|
||||
check the month's archive), or a failure: `Mismatch` (the chain was rewritten and re-hashed),
|
||||
`Regressed` (head rolled back), `MissingHead`, `MissingLink` (a chained row deleted outside the
|
||||
purge paths), `CountMismatch`.
|
||||
|
||||
| Exit code | Meaning |
|
||||
|-----------|---------|
|
||||
| `0` | Every exported head is consistent with the database |
|
||||
| `3` | At least one failure — tamper evidence |
|
||||
| `1` | No records found, bad input, `CHAIN_BUSY`, server error |
|
||||
| `2` | Authorization failure (needs `Administrator` or `Viewer`, not site-scoped) |
|
||||
|
||||
Run it with `audit verify-chain`: `verify-chain` recomputes each month's links;
|
||||
`verify-heads` proves the database still holds the heads that left the box.
|
||||
|
||||
#### `audit archive list` / `audit archive verify`
|
||||
|
||||
List and verify the AuditLog Parquet cold archives (ScadaBridge#42) that the retention
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the AuditLog hash-chain heads for off-box export and checks exported heads against
|
||||
/// the database chain (ScadaBridge#107 item 1 — the external anchor of the #41 chain).
|
||||
/// </summary>
|
||||
public interface IAuditChainHeadStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads every month's chain head with its live chained-row and tombstone counts, in one
|
||||
/// consistent snapshot (under the chain lock SHARED, so no chaining pass or purge is half-way).
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One snapshot per month that has a head, oldest first.</returns>
|
||||
Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks exported head records against the database: for each distinct exported head, does
|
||||
/// the database still carry the exported hash at the exported sequence?
|
||||
/// </summary>
|
||||
/// <param name="records">Exported records (duplicates are collapsed).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The per-record verdicts.</returns>
|
||||
Task<AuditChainHeadVerificationResult> VerifyExportedHeadsAsync(
|
||||
IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct = default);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
|
||||
|
||||
@@ -32,6 +33,15 @@ public record AuditLogQueryResult(IReadOnlyList<AuditLogEntry> Entries, int Tota
|
||||
/// <param name="Month">The month to verify, <c>YYYY-MM</c>.</param>
|
||||
public record VerifyAuditChainCommand(string Month);
|
||||
|
||||
/// <summary>
|
||||
/// Checks AuditLog chain heads exported off-box (<c>AuditLog:HashChain:HeadExport</c>,
|
||||
/// ScadaBridge#107 item 1) against the database chain: does the database still carry each
|
||||
/// exported hash at its exported sequence? Read-only; same gate as
|
||||
/// <see cref="VerifyAuditChainCommand"/>. The CLI reads the records from the export files.
|
||||
/// </summary>
|
||||
/// <param name="Records">The exported head records (at most 50,000; duplicates are collapsed).</param>
|
||||
public record VerifyAuditChainHeadsCommand(IReadOnlyList<AuditChainHeadRecord> Records);
|
||||
|
||||
/// <summary>
|
||||
/// Lists the AuditLog Parquet cold archives under the configured archive directory (audit
|
||||
/// roadmap T2, ScadaBridge#42). Read-only; OperationalAudit roles, refused for a site-scoped
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
/// <summary>
|
||||
/// One month's AuditLog hash-chain head as exported off-box (ScadaBridge#107 item 1). The
|
||||
/// export is the external anchor the in-database chain lacks: a principal able to rewrite rows,
|
||||
/// recompute every later link and update <c>AuditLogChainHead</c> produces a chain that
|
||||
/// <c>verify-chain</c> accepts, but cannot change a head hash that already left the box.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Serialised as one JSON object per line (camelCase) by both export sinks; the
|
||||
/// <c>audit verify-heads</c> CLI reads those lines back and the server compares them with the
|
||||
/// database. Hashes are lowercase hex. The format is versioned by <see cref="V"/>.
|
||||
/// </remarks>
|
||||
/// <param name="V">Record format version (1).</param>
|
||||
/// <param name="Month">Chain partition, the UTC calendar month <c>YYYY-MM</c>.</param>
|
||||
/// <param name="LastSeq">The head's last sequence number — links written for the month so far.</param>
|
||||
/// <param name="LastHash">The head's last link hash (hex).</param>
|
||||
/// <param name="PurgedThroughSeq">The partition-purge checkpoint anchor's sequence, or null.</param>
|
||||
/// <param name="PurgedThroughHash">The anchor's hash (hex), or null.</param>
|
||||
/// <param name="ChainedRows">Chained rows of the month still in <c>dbo.AuditLog</c> at export time.</param>
|
||||
/// <param name="Tombstones">Per-channel purge tombstones of the month at export time.</param>
|
||||
/// <param name="HeadUpdatedAtUtc">When the head row was last advanced.</param>
|
||||
/// <param name="ExportedAtUtc">When this record was exported.</param>
|
||||
/// <param name="Node">The central node that exported it.</param>
|
||||
public sealed record AuditChainHeadRecord(
|
||||
int V,
|
||||
string Month,
|
||||
long LastSeq,
|
||||
string LastHash,
|
||||
long? PurgedThroughSeq,
|
||||
string? PurgedThroughHash,
|
||||
long ChainedRows,
|
||||
long Tombstones,
|
||||
DateTime HeadUpdatedAtUtc,
|
||||
DateTime ExportedAtUtc,
|
||||
string? Node)
|
||||
{
|
||||
/// <summary>The current record format version.</summary>
|
||||
public const int CurrentVersion = 1;
|
||||
}
|
||||
|
||||
/// <summary>Outcome of comparing one exported head with the database chain.</summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<AuditChainHeadStatus>))]
|
||||
public enum AuditChainHeadStatus
|
||||
{
|
||||
/// <summary>The database head is exactly the exported head (same sequence, same hash, same row + tombstone count).</summary>
|
||||
Matches,
|
||||
|
||||
/// <summary>The chain has grown since the export and the link at the exported sequence still carries the exported hash.</summary>
|
||||
Extended,
|
||||
|
||||
/// <summary>
|
||||
/// The link at the exported sequence was removed by a monthly partition purge (it is at or
|
||||
/// before the checkpoint anchor and not the anchor itself), so the database cannot confirm
|
||||
/// it; verify the month's Parquet archive instead (<c>audit archive verify</c>).
|
||||
/// </summary>
|
||||
Purged,
|
||||
|
||||
/// <summary>The link at the exported sequence carries a different hash: the chain was rewritten.</summary>
|
||||
Mismatch,
|
||||
|
||||
/// <summary>The database head's sequence is BELOW the exported one: links were removed and the head rolled back.</summary>
|
||||
Regressed,
|
||||
|
||||
/// <summary>The month's head row is gone although links had been exported for it.</summary>
|
||||
MissingHead,
|
||||
|
||||
/// <summary>No row, tombstone or anchor carries the exported sequence any more: a chained row was deleted outside the purge paths.</summary>
|
||||
MissingLink,
|
||||
|
||||
/// <summary>
|
||||
/// Same head as exported, no partition purge since, but chained rows + tombstones differ
|
||||
/// from the exported counts: a chained row was removed or its link cleared.
|
||||
/// </summary>
|
||||
CountMismatch,
|
||||
}
|
||||
|
||||
/// <summary>One exported head checked against the database.</summary>
|
||||
/// <param name="Month">Chain partition (<c>YYYY-MM</c>).</param>
|
||||
/// <param name="ExportedSeq">The exported head's sequence.</param>
|
||||
/// <param name="ExportedAtUtc">When it was exported.</param>
|
||||
/// <param name="Node">Which node exported it.</param>
|
||||
/// <param name="Status">The verdict.</param>
|
||||
/// <param name="DatabaseSeq">The database head's current sequence, or null when it has none.</param>
|
||||
/// <param name="Detail">Human-readable explanation.</param>
|
||||
public sealed record AuditChainHeadCheck(
|
||||
string Month,
|
||||
long ExportedSeq,
|
||||
DateTime ExportedAtUtc,
|
||||
string? Node,
|
||||
AuditChainHeadStatus Status,
|
||||
long? DatabaseSeq,
|
||||
string Detail)
|
||||
{
|
||||
/// <summary>True for the verdicts that are tamper evidence.</summary>
|
||||
[JsonIgnore]
|
||||
public bool IsFailure => Status is AuditChainHeadStatus.Mismatch or AuditChainHeadStatus.Regressed
|
||||
or AuditChainHeadStatus.MissingHead or AuditChainHeadStatus.MissingLink or AuditChainHeadStatus.CountMismatch;
|
||||
}
|
||||
|
||||
/// <summary>Result of <c>audit verify-heads</c>.</summary>
|
||||
/// <param name="Verified">True when no check is a failure.</param>
|
||||
/// <param name="RecordsChecked">Distinct exported heads checked.</param>
|
||||
/// <param name="Checks">Every check, failures first, then by month and sequence.</param>
|
||||
public sealed record AuditChainHeadVerificationResult(bool Verified, int RecordsChecked, IReadOnlyList<AuditChainHeadCheck> Checks);
|
||||
|
||||
/// <summary>The database's current chain head for a month, as read for export.</summary>
|
||||
/// <param name="Month">First instant of the month (UTC).</param>
|
||||
/// <param name="LastSeq">Head sequence.</param>
|
||||
/// <param name="LastHash">Head hash.</param>
|
||||
/// <param name="PurgedThroughSeq">Checkpoint anchor sequence, or null.</param>
|
||||
/// <param name="PurgedThroughHash">Checkpoint anchor hash, or null.</param>
|
||||
/// <param name="ChainedRows">Chained rows of the month still present.</param>
|
||||
/// <param name="Tombstones">Tombstones of the month.</param>
|
||||
/// <param name="UpdatedAtUtc">When the head row was last advanced.</param>
|
||||
public sealed record AuditChainHeadSnapshot(
|
||||
DateTime Month,
|
||||
long LastSeq,
|
||||
byte[] LastHash,
|
||||
long? PurgedThroughSeq,
|
||||
byte[]? PurgedThroughHash,
|
||||
long ChainedRows,
|
||||
long Tombstones,
|
||||
DateTime UpdatedAtUtc);
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
using System.Globalization;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
/// <summary>
|
||||
/// Pure comparison of one exported chain head (ScadaBridge#107 item 1) with the database's
|
||||
/// current state for that month. Kept free of I/O so every verdict is unit-testable; the
|
||||
/// database side supplies the head snapshot and, when needed, the link stored at the
|
||||
/// exported sequence number.
|
||||
/// </summary>
|
||||
public static class AuditChainHeadComparer
|
||||
{
|
||||
/// <summary>What the database stores at one chain sequence number.</summary>
|
||||
/// <param name="RowHash">The <c>RowHash</c> of the live row carrying the sequence, or null.</param>
|
||||
/// <param name="TombstoneHash">The <c>RowHash</c> of a per-channel purge tombstone carrying it, or null.</param>
|
||||
public readonly record struct StoredLink(byte[]? RowHash, byte[]? TombstoneHash);
|
||||
|
||||
/// <summary>Compares one exported head with the database.</summary>
|
||||
/// <param name="record">The exported head.</param>
|
||||
/// <param name="head">The database head for the record's month, or null when there is none.</param>
|
||||
/// <param name="linkAt">Reads the stored link at a sequence number (only called when the chain has grown).</param>
|
||||
/// <returns>The verdict.</returns>
|
||||
public static async Task<AuditChainHeadCheck> CompareAsync(
|
||||
AuditChainHeadRecord record, AuditChainHeadSnapshot? head, Func<long, Task<StoredLink>> linkAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(record);
|
||||
ArgumentNullException.ThrowIfNull(linkAt);
|
||||
|
||||
AuditChainHeadCheck Result(AuditChainHeadStatus status, string detail) =>
|
||||
new(record.Month, record.LastSeq, record.ExportedAtUtc, record.Node, status, head?.LastSeq, detail);
|
||||
|
||||
if (!TryParseHex(record.LastHash, out var exported))
|
||||
{
|
||||
return Result(AuditChainHeadStatus.Mismatch, $"Exported hash '{record.LastHash}' is not 32 bytes of hex.");
|
||||
}
|
||||
|
||||
if (head is null)
|
||||
{
|
||||
return Result(AuditChainHeadStatus.MissingHead,
|
||||
$"{record.Month} has no chain head in the database, but {record.LastSeq} links were exported for it.");
|
||||
}
|
||||
|
||||
if (head.LastSeq < record.LastSeq)
|
||||
{
|
||||
return Result(AuditChainHeadStatus.Regressed,
|
||||
$"Database head is at sequence {head.LastSeq}, below the exported {record.LastSeq}: links were removed and the head rolled back.");
|
||||
}
|
||||
|
||||
if (head.LastSeq == record.LastSeq)
|
||||
{
|
||||
if (!exported.AsSpan().SequenceEqual(head.LastHash))
|
||||
{
|
||||
return Result(AuditChainHeadStatus.Mismatch,
|
||||
$"Database head hash at sequence {head.LastSeq} is {ToHex(head.LastHash)}, exported {record.LastHash}: the chain was rewritten.");
|
||||
}
|
||||
|
||||
// No link was added since the export. Unless a partition purge set (or moved) the
|
||||
// anchor since, rows + tombstones are invariant: the per-channel purge turns a row
|
||||
// into a tombstone, nothing else may remove a chained row.
|
||||
if (head.PurgedThroughSeq == record.PurgedThroughSeq
|
||||
&& head.ChainedRows + head.Tombstones != record.ChainedRows + record.Tombstones)
|
||||
{
|
||||
return Result(AuditChainHeadStatus.CountMismatch,
|
||||
$"Chained rows + tombstones are {head.ChainedRows} + {head.Tombstones} in the database, {record.ChainedRows} + {record.Tombstones} exported.");
|
||||
}
|
||||
|
||||
return Result(AuditChainHeadStatus.Matches, "Database head equals the exported head.");
|
||||
}
|
||||
|
||||
// The chain has grown: the link at the exported sequence must still carry the exported hash.
|
||||
if (head.PurgedThroughSeq is { } anchor && record.LastSeq <= anchor)
|
||||
{
|
||||
if (record.LastSeq < anchor)
|
||||
{
|
||||
return Result(AuditChainHeadStatus.Purged,
|
||||
$"Sequence {record.LastSeq} was removed by a partition purge (anchor at {anchor}); verify the month's archive.");
|
||||
}
|
||||
|
||||
return head.PurgedThroughHash is { } anchorHash && exported.AsSpan().SequenceEqual(anchorHash)
|
||||
? Result(AuditChainHeadStatus.Extended, $"Exported head is the partition-purge anchor at {anchor}.")
|
||||
: Result(AuditChainHeadStatus.Mismatch, $"Partition-purge anchor at {anchor} does not carry the exported hash.");
|
||||
}
|
||||
|
||||
var link = await linkAt(record.LastSeq).ConfigureAwait(false);
|
||||
var stored = link.RowHash ?? link.TombstoneHash;
|
||||
if (stored is null)
|
||||
{
|
||||
return Result(AuditChainHeadStatus.MissingLink,
|
||||
$"No row, tombstone or anchor carries sequence {record.LastSeq}: a chained row was deleted outside the purge paths.");
|
||||
}
|
||||
|
||||
return exported.AsSpan().SequenceEqual(stored)
|
||||
? Result(AuditChainHeadStatus.Extended,
|
||||
$"Chain grew to {head.LastSeq}; the {(link.RowHash is null ? "tombstone" : "row")} at {record.LastSeq} still carries the exported hash.")
|
||||
: Result(AuditChainHeadStatus.Mismatch,
|
||||
$"The {(link.RowHash is null ? "tombstone" : "row")} at sequence {record.LastSeq} carries {ToHex(stored)}, exported {record.LastHash}: the chain was rewritten.");
|
||||
}
|
||||
|
||||
/// <summary>Lowercase hex of a hash.</summary>
|
||||
/// <param name="bytes">The bytes.</param>
|
||||
/// <returns>Hex string.</returns>
|
||||
public static string ToHex(byte[] bytes) => Convert.ToHexString(bytes).ToLowerInvariant();
|
||||
|
||||
/// <summary>Parses a <c>YYYY-MM</c> month into its first instant (UTC).</summary>
|
||||
/// <param name="month">The month text.</param>
|
||||
/// <param name="monthStart">The parsed month.</param>
|
||||
/// <returns>True when well-formed.</returns>
|
||||
public static bool TryParseMonth(string? month, out DateTime monthStart)
|
||||
{
|
||||
if (DateTime.TryParseExact(month, "yyyy-MM", CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsed))
|
||||
{
|
||||
monthStart = new DateTime(parsed.Year, parsed.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
return true;
|
||||
}
|
||||
|
||||
monthStart = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryParseHex(string? hex, out byte[] bytes)
|
||||
{
|
||||
bytes = [];
|
||||
if (hex is null || hex.Length != AuditChainHasher.HashLength * 2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
bytes = Convert.FromHexString(hex);
|
||||
return true;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ public static class AuditChainLock
|
||||
/// those two columns; the ingest writer role keeps its table-wide DENY UPDATE.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class AuditLogHashChain : IAuditLogHashChain
|
||||
public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
{
|
||||
// 4 parameters per row; 200 rows = 800 parameters, far under SQL Server's 2,100 cap.
|
||||
private const int UpdateChunkRows = 200;
|
||||
@@ -273,6 +273,151 @@ public sealed class AuditLogHashChain : IAuditLogHashChain
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsAsync(CancellationToken ct = default)
|
||||
{
|
||||
var conn = _context.Database.GetDbConnection();
|
||||
var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false);
|
||||
await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false);
|
||||
var heads = await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false);
|
||||
await tx.CommitAsync(ct).ConfigureAwait(false);
|
||||
return heads;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (openedHere)
|
||||
{
|
||||
await conn.CloseAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<AuditChainHeadVerificationResult> VerifyExportedHeadsAsync(
|
||||
IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(records);
|
||||
|
||||
// Collapse duplicates (both centrals export, and every unchanged re-export repeats a
|
||||
// record): one check per distinct (month, sequence, hash) claim.
|
||||
var distinct = records
|
||||
.GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash?.ToLowerInvariant()))
|
||||
.Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First())
|
||||
.ToList();
|
||||
|
||||
var conn = _context.Database.GetDbConnection();
|
||||
var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false);
|
||||
await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false);
|
||||
var heads = (await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false))
|
||||
.ToDictionary(h => AuditChainHasher.PartitionLabel(h.Month), StringComparer.Ordinal);
|
||||
|
||||
var checks = new List<AuditChainHeadCheck>(distinct.Count);
|
||||
foreach (var record in distinct)
|
||||
{
|
||||
if (!AuditChainHeadComparer.TryParseMonth(record.Month, out var month))
|
||||
{
|
||||
checks.Add(new AuditChainHeadCheck(record.Month ?? "", record.LastSeq, record.ExportedAtUtc, record.Node,
|
||||
AuditChainHeadStatus.Mismatch, null, $"Exported month '{record.Month}' is not YYYY-MM."));
|
||||
continue;
|
||||
}
|
||||
|
||||
heads.TryGetValue(AuditChainHasher.PartitionLabel(month), out var head);
|
||||
checks.Add(await AuditChainHeadComparer.CompareAsync(
|
||||
record, head, seq => ReadStoredLinkAsync(conn, tx, month, seq, ct)).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
await tx.CommitAsync(ct).ConfigureAwait(false);
|
||||
var ordered = checks
|
||||
.OrderByDescending(c => c.IsFailure)
|
||||
.ThenBy(c => c.Month, StringComparer.Ordinal)
|
||||
.ThenBy(c => c.ExportedSeq)
|
||||
.ToList();
|
||||
return new AuditChainHeadVerificationResult(ordered.All(c => !c.IsFailure), ordered.Count, ordered);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (openedHere)
|
||||
{
|
||||
await conn.CloseAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AcquireSharedOrThrowAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
|
||||
{
|
||||
if (!await TryAcquireLockAsync(conn, tx, "Shared", VerifyLockTimeout, ct).ConfigureAwait(false))
|
||||
{
|
||||
await tx.RollbackAsync(ct).ConfigureAwait(false);
|
||||
throw new TimeoutException(
|
||||
$"The AuditLog hash chain was busy (chaining or purge in progress) for {VerifyLockTimeout.TotalSeconds:0}s; retry.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsCoreAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
|
||||
{
|
||||
// Per month: the head row, the chained rows still present (the filtered ChainSeq index,
|
||||
// partition-eliminated by the month range) and the tombstones. The head table holds one
|
||||
// row per month ever chained, so this is a few dozen rows.
|
||||
var heads = new List<AuditChainHeadSnapshot>();
|
||||
await using var cmd = Command(conn, tx, @"
|
||||
SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughHash, h.UpdatedAtUtc,
|
||||
(SELECT COUNT_BIG(*) FROM dbo.AuditLog a
|
||||
WHERE a.ChainSeq IS NOT NULL
|
||||
AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7))
|
||||
AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows,
|
||||
(SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones
|
||||
FROM dbo.AuditLogChainHead h
|
||||
ORDER BY h.ChainMonth;");
|
||||
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
heads.Add(new AuditChainHeadSnapshot(
|
||||
DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc),
|
||||
reader.GetInt64(1),
|
||||
(byte[])reader.GetValue(2),
|
||||
reader.IsDBNull(3) ? null : reader.GetInt64(3),
|
||||
reader.IsDBNull(4) ? null : (byte[])reader.GetValue(4),
|
||||
reader.GetInt64(6),
|
||||
reader.GetInt64(7),
|
||||
DateTime.SpecifyKind(reader.GetDateTime(5), DateTimeKind.Utc)));
|
||||
}
|
||||
|
||||
return heads;
|
||||
}
|
||||
|
||||
private async Task<AuditChainHeadComparer.StoredLink> ReadStoredLinkAsync(
|
||||
DbConnection conn, DbTransaction tx, DateTime month, long seq, CancellationToken ct)
|
||||
{
|
||||
byte[]? rowHash;
|
||||
await using (var cmd = Command(conn, tx,
|
||||
"SELECT TOP (1) RowHash FROM dbo.AuditLog WHERE ChainSeq = @seq AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;"))
|
||||
{
|
||||
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
|
||||
cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq });
|
||||
AddMonthRange(cmd, month);
|
||||
rowHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[];
|
||||
}
|
||||
|
||||
byte[]? tombstoneHash = null;
|
||||
if (rowHash is null)
|
||||
{
|
||||
await using var cmd = Command(conn, tx,
|
||||
"SELECT RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month AND ChainSeq = @seq;");
|
||||
AddDate(cmd, "@month", month);
|
||||
cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq });
|
||||
tombstoneHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[];
|
||||
}
|
||||
|
||||
return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes the chain application lock inside <paramref name="tx"/> (released at commit /
|
||||
/// rollback). Returns false when the lock could not be granted within <paramref name="timeout"/>.
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
/// <summary>
|
||||
/// The connection the AuditLog maintenance paths run on — the hash-chain job and both purge
|
||||
/// paths (partition switch-out with its archive, per-channel DELETE) — ScadaBridge#107 item 3.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// With <c>AuditLog:MaintenanceConnectionString</c> unset (the default), this is the scope's
|
||||
/// ordinary <see cref="ScadaBridgeDbContext"/>: one login for everything, exactly as before.
|
||||
/// With it set, the maintenance paths get their OWN context on that connection string, so a
|
||||
/// DBA can give the runtime (writer) login only <c>scadabridge_audit_writer</c> — whose DENYs
|
||||
/// stop it writing chain state or deleting audit rows — and a separate maintenance login the
|
||||
/// <c>scadabridge_audit_chainer</c> + <c>scadabridge_audit_purger</c> roles. The string is read
|
||||
/// after the pre-host <c>${secret:}</c> expansion, so it may (and in production should) be a
|
||||
/// secret reference.
|
||||
/// </para>
|
||||
/// <para>Scoped: an owned context is disposed with the scope; a borrowed one is left to DI.</para>
|
||||
/// </remarks>
|
||||
public sealed class AuditMaintenanceContext : IAsyncDisposable, IDisposable
|
||||
{
|
||||
/// <summary>DI key of the maintenance-login <c>IAuditLogRepository</c> and <c>IAuditLogArchive</c>.</summary>
|
||||
public const string ServiceKey = "audit-maintenance";
|
||||
|
||||
private readonly bool _owned;
|
||||
|
||||
private AuditMaintenanceContext(ScadaBridgeDbContext context, bool owned)
|
||||
{
|
||||
Context = context;
|
||||
_owned = owned;
|
||||
}
|
||||
|
||||
/// <summary>The context to run maintenance SQL on.</summary>
|
||||
public ScadaBridgeDbContext Context { get; }
|
||||
|
||||
/// <summary>True when <see cref="Context"/> is on the separate maintenance connection string.</summary>
|
||||
public bool UsesSeparateLogin => _owned;
|
||||
|
||||
/// <summary>Wraps the scope's ordinary context (no separate maintenance login configured).</summary>
|
||||
/// <param name="context">The scope's context.</param>
|
||||
/// <returns>The wrapper.</returns>
|
||||
public static AuditMaintenanceContext Shared(ScadaBridgeDbContext context)
|
||||
=> new(context ?? throw new ArgumentNullException(nameof(context)), owned: false);
|
||||
|
||||
/// <summary>Opens an owned context on the maintenance connection string.</summary>
|
||||
/// <param name="connectionString">The maintenance connection string.</param>
|
||||
/// <param name="protectionProvider">Data protection for secret-bearing columns (same as the runtime context).</param>
|
||||
/// <returns>The wrapper; disposing it disposes the context.</returns>
|
||||
public static AuditMaintenanceContext Separate(string connectionString, IDataProtectionProvider? protectionProvider)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(connectionString);
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(connectionString, sql => sql.EnableRetryOnFailure(
|
||||
maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(30), errorNumbersToAdd: null))
|
||||
.ConfigureWarnings(w => w.Ignore(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
var context = protectionProvider is null
|
||||
? new ScadaBridgeDbContext(options)
|
||||
: new ScadaBridgeDbContext(options, protectionProvider);
|
||||
return new AuditMaintenanceContext(context, owned: true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync() => _owned ? Context.DisposeAsync() : ValueTask.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
if (_owned)
|
||||
{
|
||||
Context.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The configured maintenance connection string (null = use the runtime connection).</summary>
|
||||
/// <param name="ConnectionString">The connection string, or null.</param>
|
||||
public sealed record AuditMaintenanceConnection(string? ConnectionString);
|
||||
@@ -20,8 +20,14 @@ public static class ServiceCollectionExtensions
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection to register into.</param>
|
||||
/// <param name="connectionString">SQL Server connection string for the central configuration database.</param>
|
||||
/// <param name="auditMaintenanceConnectionString">
|
||||
/// Optional separate connection string for the AuditLog maintenance paths — hash-chain job
|
||||
/// and both purge paths (<c>AuditLog:MaintenanceConnectionString</c>, ScadaBridge#107 item 3).
|
||||
/// Null or blank runs them on <paramref name="connectionString"/> as before.
|
||||
/// </param>
|
||||
/// <returns>The same <paramref name="services"/> collection, for chaining.</returns>
|
||||
public static IServiceCollection AddConfigurationDatabase(this IServiceCollection services, string connectionString)
|
||||
public static IServiceCollection AddConfigurationDatabase(
|
||||
this IServiceCollection services, string connectionString, string? auditMaintenanceConnectionString = null)
|
||||
{
|
||||
// The DbContext is constructed via the (options, IDataProtectionProvider) overload so
|
||||
// secret-bearing configuration columns are encrypted at rest. AddDataProtection below
|
||||
@@ -111,9 +117,31 @@ public static class ServiceCollectionExtensions
|
||||
// fresh scope on each tick (mirrors AuditLogPurgeActor / AuditLogIngestActor).
|
||||
services.AddScoped<IPartitionMaintenance, AuditLogPartitionMaintenance>();
|
||||
|
||||
// AuditLog maintenance connection (ScadaBridge#107 item 3): the chain job and both purge
|
||||
// paths run on AuditMaintenanceContext — the scope's own context unless a separate
|
||||
// maintenance login is configured, in which case an owned context on that string.
|
||||
services.AddSingleton(new AuditMaintenanceConnection(
|
||||
string.IsNullOrWhiteSpace(auditMaintenanceConnectionString) ? null : auditMaintenanceConnectionString));
|
||||
services.AddScoped(sp =>
|
||||
{
|
||||
var maintenance = sp.GetRequiredService<AuditMaintenanceConnection>().ConnectionString;
|
||||
return maintenance is null
|
||||
? AuditMaintenanceContext.Shared(sp.GetRequiredService<ScadaBridgeDbContext>())
|
||||
: AuditMaintenanceContext.Separate(maintenance, sp.GetService<IDataProtectionProvider>());
|
||||
});
|
||||
services.AddKeyedScoped<IAuditLogRepository>(AuditMaintenanceContext.ServiceKey, (sp, _) => new AuditLogRepository(
|
||||
sp.GetRequiredService<AuditMaintenanceContext>().Context,
|
||||
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogRepository>>()));
|
||||
|
||||
// AuditLog hash-chain tamper evidence (ScadaBridge#41): chained by the central
|
||||
// AuditLogHashChainService, verified by the VerifyAuditChain management command.
|
||||
services.AddScoped<IAuditLogHashChain, AuditLogHashChain>();
|
||||
// AuditLogHashChainService, verified by the VerifyAuditChain management command; the
|
||||
// same instance reads/verifies the off-box head export (#107 item 1). Both on the
|
||||
// maintenance connection.
|
||||
services.AddScoped(sp => new AuditLogHashChain(
|
||||
sp.GetRequiredService<AuditMaintenanceContext>().Context,
|
||||
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogHashChain>>()));
|
||||
services.AddScoped<IAuditLogHashChain>(sp => sp.GetRequiredService<AuditLogHashChain>());
|
||||
services.AddScoped<IAuditChainHeadStore>(sp => sp.GetRequiredService<AuditLogHashChain>());
|
||||
|
||||
// AuditLog Parquet cold archive (ScadaBridge#42): written by the purge actor before a
|
||||
// partition switch, listed/verified by the ListAuditArchives / VerifyAuditArchive
|
||||
@@ -123,6 +151,12 @@ public static class ServiceCollectionExtensions
|
||||
sp.GetRequiredService<ScadaBridgeDbContext>(),
|
||||
sp.GetService<Microsoft.Extensions.Options.IOptions<AuditLogArchiveOptions>>()?.Value ?? new AuditLogArchiveOptions(),
|
||||
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogPartitionArchive>>()));
|
||||
// The purge actor's archive-then-purge runs on the maintenance connection (#107 item 3);
|
||||
// list/verify from the management surface stay on the runtime connection (reads only).
|
||||
services.AddKeyedScoped<IAuditLogArchive>(AuditMaintenanceContext.ServiceKey, (sp, _) => new AuditLogPartitionArchive(
|
||||
sp.GetRequiredService<AuditMaintenanceContext>().Context,
|
||||
sp.GetService<Microsoft.Extensions.Options.IOptions<AuditLogArchiveOptions>>()?.Value ?? new AuditLogArchiveOptions(),
|
||||
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogPartitionArchive>>()));
|
||||
|
||||
// Opt-in config-audit retention (ScadaBridge:ConfigAudit:RetentionDays, default 0 = off).
|
||||
// Options binding to configuration is done by the host; without it the defaults apply
|
||||
|
||||
@@ -531,7 +531,12 @@ try
|
||||
|
||||
var configDbConnectionString = configuration["ScadaBridge:Database:ConfigurationDb"]
|
||||
?? throw new InvalidOperationException("ScadaBridge:Database:ConfigurationDb connection string is required for Central role.");
|
||||
builder.Services.AddConfigurationDatabase(configDbConnectionString);
|
||||
// Optional second login for the AuditLog maintenance paths (chain job + both purge
|
||||
// paths, ScadaBridge#107 item 3). Read from the pre-host-expanded configuration, so it
|
||||
// may be a ${secret:} reference; unset = everything on the runtime connection, as before.
|
||||
builder.Services.AddConfigurationDatabase(
|
||||
configDbConnectionString,
|
||||
configuration[ZB.MOM.WW.ScadaBridge.AuditLog.ServiceCollectionExtensions.MaintenanceConnectionStringKey]);
|
||||
builder.Services.AddOptions<ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.ConfigAuditOptions>()
|
||||
.Bind(configuration.GetSection(ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.ConfigAuditOptions.SectionName))
|
||||
.ValidateOnStart();
|
||||
|
||||
@@ -392,6 +392,8 @@ public class ManagementActor : ReceiveActor
|
||||
// permission). A site-scoped caller is refused in the handler: a month's chain
|
||||
// spans every site.
|
||||
VerifyAuditChainCommand => OperationalAuditReaders,
|
||||
// Off-box chain-head check (ScadaBridge#107 item 1): the same audit read.
|
||||
VerifyAuditChainHeadsCommand => OperationalAuditReaders,
|
||||
|
||||
// AuditLog Parquet archives (ScadaBridge#42): audit reads with verify-chain's gate —
|
||||
// OperationalAudit roles, site-scoped callers refused in the handler.
|
||||
@@ -569,6 +571,7 @@ public class ManagementActor : ReceiveActor
|
||||
// Audit Log
|
||||
QueryAuditLogCommand cmd => await HandleQueryAuditLog(sp, cmd),
|
||||
VerifyAuditChainCommand cmd => await HandleVerifyAuditChain(sp, cmd, user),
|
||||
VerifyAuditChainHeadsCommand cmd => await HandleVerifyAuditChainHeads(sp, cmd, user),
|
||||
ListAuditArchivesCommand => await HandleListAuditArchives(sp, user),
|
||||
VerifyAuditArchiveCommand cmd => await HandleVerifyAuditArchive(sp, cmd, user),
|
||||
|
||||
@@ -3273,6 +3276,48 @@ public class ManagementActor : ReceiveActor
|
||||
return new AuditArchiveListResult(archive.IsEnabled, await archive.ListAsync());
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on records per <see cref="VerifyAuditChainHeadsCommand"/>.</summary>
|
||||
internal const int MaxChainHeadRecords = 50_000;
|
||||
|
||||
/// <summary>
|
||||
/// Checks exported AuditLog chain heads against the database (ScadaBridge#107 item 1). Same
|
||||
/// gate as verify-chain: a site-scoped, non-Administrator caller is refused because the
|
||||
/// chain spans every site.
|
||||
/// </summary>
|
||||
private static async Task<object?> HandleVerifyAuditChainHeads(
|
||||
IServiceProvider sp, VerifyAuditChainHeadsCommand cmd, AuthenticatedUser user)
|
||||
{
|
||||
if (cmd.Records is null || cmd.Records.Count == 0)
|
||||
{
|
||||
throw new ManagementCommandException("No exported chain-head records were supplied.", "INVALID_ARGUMENT");
|
||||
}
|
||||
|
||||
if (cmd.Records.Count > MaxChainHeadRecords)
|
||||
{
|
||||
throw new ManagementCommandException(
|
||||
$"{cmd.Records.Count} records exceed the {MaxChainHeadRecords} limit; verify fewer export files at a time.",
|
||||
"INVALID_ARGUMENT");
|
||||
}
|
||||
|
||||
if (user.PermittedSiteIds.Length > 0
|
||||
&& !user.Roles.Contains(Roles.Administrator, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new SiteScopeViolationException(
|
||||
"Access denied: the audit hash chain spans every site; verify-heads needs an "
|
||||
+ "OperationalAudit role that is not scoped to specific sites.");
|
||||
}
|
||||
|
||||
var store = sp.GetRequiredService<IAuditChainHeadStore>();
|
||||
try
|
||||
{
|
||||
return await store.VerifyExportedHeadsAsync(cmd.Records);
|
||||
}
|
||||
catch (TimeoutException ex)
|
||||
{
|
||||
throw new ManagementCommandException(ex.Message, "CHAIN_BUSY");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies one AuditLog Parquet archive (ScadaBridge#42): files against the manifest,
|
||||
/// the chain recomputed from the archived rows, the chain end against the purge anchor.
|
||||
|
||||
@@ -63,6 +63,9 @@ public static class ManagementEndpoints
|
||||
// AuditLog hash-chain verification (ScadaBridge#41) recomputes one hash per row of
|
||||
// a whole month — millions of rows on a busy central.
|
||||
typeof(VerifyAuditChainCommand),
|
||||
// Off-box chain-head check (ScadaBridge#107): one link lookup per exported head, under
|
||||
// the chain lock SHARED (waits behind a chaining pass).
|
||||
typeof(VerifyAuditChainHeadsCommand),
|
||||
// AuditLog archive verification (ScadaBridge#42) hashes and re-chains a whole month's
|
||||
// Parquet archive.
|
||||
typeof(VerifyAuditArchiveCommand),
|
||||
|
||||
Reference in New Issue
Block a user