feat(audit): off-box chain-head export, verify-heads, maintenance login (ScadaBridge#107)

Item 1 (owner decision: no signing). AuditLog:HashChain:HeadExport (default
off): both centrals periodically read every month's chain head (LastSeq,
LastHash, purge anchor, chained rows, tombstones; one snapshot under the
chain lock SHARED) and append the changed heads to a sink the DB login cannot
write - File (append-only JSONL per node per day, local or UNC) or Syslog
(RFC 5424 over UDP / TCP octet counting). `audit verify-heads --path` reads
those records back (file or syslog-collector lines) and the new
VerifyAuditChainHeads command checks each against the database: Matches /
Extended / Purged, or Mismatch / Regressed / MissingHead / MissingLink /
CountMismatch (exit 3). AuditChainHeadStoreTests forges a chain with the
production hasher: verify-chain says Intact, verify-heads says Mismatch.

Item 3. AuditLog:MaintenanceConnectionString (${secret:}-capable, unset =
today's single login) runs the chain job, chain verify / head export, the
partition switch-out + archive and the per-channel purge on
AuditMaintenanceContext (keyed audit-maintenance repository/archive, purge
actor prefers them). AuditMaintenanceLoginTests runs the hardened setup on
SQL Server: writer (datareader/datawriter/audit_writer) ingests and is
refused chaining and deleting; the chainer+purger login does the full cycle.
Two-login grants documented in Component-AuditLog.md.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 23:16:05 -05:00
parent 5e7d5389d4
commit c4d7ec9b96
40 changed files with 2769 additions and 43 deletions
@@ -792,4 +792,34 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
StopPurgeActor(actor);
}
/// <summary>
/// ScadaBridge#107 item 3: when the host registered the maintenance-connection repository
/// (keyed <c>audit-maintenance</c>), the purge runs on it, never on the runtime one.
/// </summary>
[Fact]
public void Tick_UsesTheMaintenanceConnectionRepository_WhenRegistered()
{
var runtime = new RecordingRepo { Boundaries = [new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc)] };
var maintenance = new RecordingRepo { Boundaries = [new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc)] };
var services = new ServiceCollection();
services.AddScoped<IAuditLogRepository>(_ => runtime);
services.AddKeyedScoped<IAuditLogRepository>(
ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.AuditMaintenanceContext.ServiceKey, (_, _) => maintenance);
var sp = services.BuildServiceProvider();
var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
sp,
Options.Create(FastTickOptions()),
Options.Create(new AuditLogOptions()),
NullLogger<AuditLogPurgeActor>.Instance)));
AwaitAssert(
() => Assert.NotEmpty(maintenance.SwitchedBoundaries),
duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50));
Assert.Empty(runtime.ThresholdQueries);
Assert.Empty(runtime.SwitchedBoundaries);
StopPurgeActor(actor);
}
}
@@ -0,0 +1,164 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central.HeadExport;
/// <summary>
/// <see cref="AuditChainHeadExportService"/> (ScadaBridge#107 item 1): first tick exports every
/// head, later ticks only the changed ones, a failed write is retried with the same heads, and
/// the options validator guards each sink's settings.
/// </summary>
public sealed class AuditChainHeadExportServiceTests
{
private sealed class FakeStore : IAuditChainHeadStore
{
public List<AuditChainHeadSnapshot> Heads { get; } = [];
public Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsAsync(CancellationToken ct = default)
=> Task.FromResult<IReadOnlyList<AuditChainHeadSnapshot>>(Heads.ToList());
public Task<AuditChainHeadVerificationResult> VerifyExportedHeadsAsync(
IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct = default) => throw new NotSupportedException();
}
private sealed class RecordingSink : IAuditChainHeadSink
{
public List<IReadOnlyList<AuditChainHeadRecord>> Writes { get; } = [];
public bool Fail { get; set; }
public string Describe => "recording";
public Task WriteAsync(IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct)
{
if (Fail)
{
throw new IOException("share offline");
}
Writes.Add(records);
return Task.CompletedTask;
}
}
private static AuditChainHeadSnapshot Head(int month, long seq, long rows) => new(
new DateTime(2026, month, 1, 0, 0, 0, DateTimeKind.Utc), seq, Enumerable.Repeat((byte)seq, 32).ToArray(),
null, null, rows, 0, new DateTime(2026, month, 2, 0, 0, 0, DateTimeKind.Utc));
private static (AuditChainHeadExportService Service, FakeStore Store, RecordingSink Sink) Create(bool exportUnchanged = false)
{
var store = new FakeStore();
var sink = new RecordingSink();
var services = new ServiceCollection();
services.AddScoped<IAuditChainHeadStore>(_ => store);
var options = Options.Create(new AuditLogHashChainOptions
{
HeadExport = new AuditChainHeadExportOptions { Enabled = true, ExportUnchanged = exportUnchanged },
});
var service = new AuditChainHeadExportService(
services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>(),
options, sink, "central-a", NullLogger<AuditChainHeadExportService>.Instance);
return (service, store, sink);
}
/// <summary>First tick: every head; next tick: only the head that moved; unchanged: nothing.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Tick_ExportsAllHeadsFirst_ThenOnlyChangedOnes()
{
var (service, store, sink) = Create();
store.Heads.AddRange([Head(4, 10, 10), Head(5, 20, 20)]);
Assert.Equal(2, await service.TickAsync(default));
Assert.Equal(0, await service.TickAsync(default));
store.Heads[1] = Head(5, 25, 25);
Assert.Equal(1, await service.TickAsync(default));
// A per-channel purge (row -> tombstone) keeps rows + tombstones, so it is not a change;
// a row disappearing without a tombstone is.
store.Heads[0] = Head(4, 10, 9);
Assert.Equal(1, await service.TickAsync(default));
Assert.Equal(3, sink.Writes.Count);
Assert.Equal(["2026-04", "2026-05"], sink.Writes[0].Select(r => r.Month));
var moved = Assert.Single(sink.Writes[1]);
Assert.Equal(("2026-05", 25L, string.Concat(Enumerable.Repeat("19", 32))), (moved.Month, moved.LastSeq, moved.LastHash));
Assert.Equal("central-a", moved.Node);
Assert.Equal(AuditChainHeadRecord.CurrentVersion, moved.V);
}
/// <summary>ExportUnchanged re-exports every head every tick.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Tick_WithExportUnchanged_ExportsEveryHeadEveryTick()
{
var (service, store, _) = Create(exportUnchanged: true);
store.Heads.AddRange([Head(4, 10, 10), Head(5, 20, 20)]);
Assert.Equal(2, await service.TickAsync(default));
Assert.Equal(2, await service.TickAsync(default));
}
/// <summary>A failed write is contained and the same heads go out on the next tick.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Tick_AfterAFailedWrite_RetriesTheSameHeads()
{
var (service, store, sink) = Create();
store.Heads.Add(Head(4, 10, 10));
sink.Fail = true;
Assert.Equal(0, await service.TickAsync(default));
sink.Fail = false;
Assert.Equal(1, await service.TickAsync(default));
Assert.Equal(10, Assert.Single(Assert.Single(sink.Writes)).LastSeq);
}
/// <summary>The validator ignores a disabled export and checks each sink's settings when enabled.</summary>
/// <param name="configure">Which settings to apply.</param>
/// <param name="expectedError">A fragment of the expected failure, or null when valid.</param>
[Theory]
[InlineData("disabled-bad", null)]
[InlineData("file-ok", null)]
[InlineData("file-relative", "File:Directory")]
[InlineData("file-missing", "File:Directory")]
[InlineData("syslog-ok", null)]
[InlineData("syslog-no-host", "Syslog:Host")]
[InlineData("syslog-bad-port", "Syslog:Port")]
[InlineData("syslog-bad-facility", "Syslog:Facility")]
[InlineData("interval-zero", "IntervalMinutes")]
public void Validator_ChecksTheSelectedSink(string configure, string? expectedError)
{
var root = OperatingSystem.IsWindows() ? @"C:\heads" : "/var/heads";
var export = configure switch
{
"disabled-bad" => new AuditChainHeadExportOptions { Enabled = false, IntervalMinutes = 0 },
"file-ok" => new AuditChainHeadExportOptions { Enabled = true, File = { Directory = root } },
"file-relative" => new AuditChainHeadExportOptions { Enabled = true, File = { Directory = "heads" } },
"file-missing" => new AuditChainHeadExportOptions { Enabled = true },
"syslog-ok" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "collector" } },
"syslog-no-host" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog },
"syslog-bad-port" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "c", Port = 0 } },
"syslog-bad-facility" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "c", Facility = 24 } },
_ => new AuditChainHeadExportOptions { Enabled = true, IntervalMinutes = 0, File = { Directory = root } },
};
var result = new AuditLogHashChainOptionsValidator().Validate(null, new AuditLogHashChainOptions { HeadExport = export });
if (expectedError is null)
{
Assert.True(result.Succeeded, result.FailureMessage);
}
else
{
Assert.True(result.Failed);
Assert.Contains(expectedError, result.FailureMessage);
}
}
}
@@ -0,0 +1,135 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central.HeadExport;
/// <summary>
/// The two off-box chain-head sinks (ScadaBridge#107 item 1): the append-only JSON-lines file
/// sink, and RFC 5424 syslog over UDP and TCP (octet-counted framing).
/// </summary>
public sealed class AuditChainHeadSinkTests : IDisposable
{
private readonly DirectoryInfo _dir = Directory.CreateTempSubdirectory("head-sink-");
/// <inheritdoc />
public void Dispose() => _dir.Delete(recursive: true);
private static AuditChainHeadRecord Record(string month, long seq) => new(
1, month, seq, new string('a', 64), null, null, seq, 0,
new DateTime(2026, 5, 3, 0, 0, 0, DateTimeKind.Utc), new DateTime(2026, 5, 3, 1, 0, 0, DateTimeKind.Utc), "central-a");
private sealed class FixedClock(DateTimeOffset now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => now;
}
/// <summary>
/// Writes append to the node's file for the day — never truncate it — one JSON record per
/// line, and a pre-existing line survives.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task FileSink_AppendsJsonLines_ToThePerNodePerDayFile_WithoutTruncating()
{
var clock = new FixedClock(new DateTimeOffset(2026, 5, 3, 1, 0, 0, TimeSpan.Zero));
var sink = new FileAuditChainHeadSink(_dir.FullName, "central-a", clock);
var path = sink.PathFor(clock.GetUtcNow().UtcDateTime);
Assert.Equal(Path.Combine(_dir.FullName, "2026", "audit-chain-heads-2026-05-03-central-a.jsonl"), path);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
await File.WriteAllTextAsync(path, "earlier line\n");
await sink.WriteAsync([Record("2026-04", 3), Record("2026-05", 7)], default);
await sink.WriteAsync([Record("2026-05", 9)], default);
var lines = await File.ReadAllLinesAsync(path);
Assert.Equal(4, lines.Length);
Assert.Equal("earlier line", lines[0]);
var parsed = lines.Skip(1).Select(l => JsonSerializer.Deserialize<AuditChainHeadRecord>(l, new JsonSerializerOptions(JsonSerializerDefaults.Web))!).ToList();
Assert.Equal([3L, 7L, 9L], parsed.Select(r => r.LastSeq));
Assert.Equal(Record("2026-05", 9), parsed[2]);
}
/// <summary>A relative export directory is refused.</summary>
[Fact]
public void FileSink_RejectsARelativeDirectory()
{
Assert.Throws<ArgumentException>(() => new FileAuditChainHeadSink("relative/dir", "central-a"));
}
/// <summary>UDP: one RFC 5424 datagram per record — PRI, version, MSGID, nil SD, JSON body.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SyslogSink_Udp_SendsOneRfc5424DatagramPerRecord()
{
using var server = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0));
var port = ((IPEndPoint)server.Client.LocalEndPoint!).Port;
var sink = new SyslogAuditChainHeadSink(
new AuditChainHeadSyslogSinkOptions { Host = "127.0.0.1", Port = port, Protocol = AuditSyslogProtocol.Udp, Facility = 13 },
hostName: "central a", time: new FixedClock(new DateTimeOffset(2026, 5, 3, 1, 2, 3, 456, TimeSpan.Zero)));
await sink.WriteAsync([Record("2026-04", 3), Record("2026-05", 7)], default);
var first = Encoding.UTF8.GetString((await server.ReceiveAsync().WaitAsync(TimeSpan.FromSeconds(5))).Buffer);
var second = Encoding.UTF8.GetString((await server.ReceiveAsync().WaitAsync(TimeSpan.FromSeconds(5))).Buffer);
// facility 13 (log audit) * 8 + severity 5 (notice) = 109; the space in the host name is dropped.
Assert.StartsWith($"<109>1 2026-05-03T01:02:03.456Z centrala scadabridge {Environment.ProcessId} chainhead - {{", first);
AssertRecordBody(first, 3);
AssertRecordBody(second, 7);
}
/// <summary>TCP: octet-counted frames (RFC 6587), one per record, on one connection.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SyslogSink_Tcp_SendsOctetCountedFrames()
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
try
{
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
var received = Task.Run(async () =>
{
using var client = await listener.AcceptTcpClientAsync();
using var ms = new MemoryStream();
await client.GetStream().CopyToAsync(ms);
return ms.ToArray();
});
var sink = new SyslogAuditChainHeadSink(
new AuditChainHeadSyslogSinkOptions { Host = "127.0.0.1", Port = port, Protocol = AuditSyslogProtocol.Tcp });
await sink.WriteAsync([Record("2026-04", 3), Record("2026-05", 7)], default);
var bytes = await received.WaitAsync(TimeSpan.FromSeconds(5));
var frames = new List<string>();
var i = 0;
while (i < bytes.Length)
{
var space = Array.IndexOf(bytes, (byte)' ', i);
var length = int.Parse(Encoding.ASCII.GetString(bytes, i, space - i));
frames.Add(Encoding.UTF8.GetString(bytes, space + 1, length));
i = space + 1 + length;
}
Assert.Equal(2, frames.Count);
Assert.All(frames, f => Assert.StartsWith("<109>1 ", f));
AssertRecordBody(frames[0], 3);
AssertRecordBody(frames[1], 7);
}
finally
{
listener.Stop();
}
}
private static void AssertRecordBody(string message, long seq)
{
var json = message[message.IndexOf('{')..];
var record = JsonSerializer.Deserialize<AuditChainHeadRecord>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!;
Assert.Equal(seq, record.LastSeq);
Assert.Equal(new string('a', 64), record.LastHash);
}
}
@@ -27,14 +27,14 @@ public class AuditCommandsScaffoldTests
Assert.False(string.IsNullOrWhiteSpace(audit.Description));
}
/// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain, backfill-source-node and archive subcommands.</summary>
/// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain, verify-heads, backfill-source-node and archive subcommands.</summary>
[Fact]
public void Audit_HasSixSubcommands_QueryExportTreeVerifyChainBackfillSourceNodeArchive()
public void Audit_HasSevenSubcommands_QueryExportTreeVerifyChainVerifyHeadsBackfillSourceNodeArchive()
{
var audit = BuildAudit();
var names = audit.Subcommands.Select(c => c.Name).OrderBy(n => n).ToArray();
Assert.Equal(
new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain" },
new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain", "verify-heads" },
names);
}
@@ -65,7 +65,7 @@ public class AuditCommandsScaffoldTests
// `archive` is a group (list / verify, ScadaBridge#42): check its leaves, not it.
var audit = BuildAudit();
var leaves = audit.Subcommands.SelectMany(c => c.Subcommands.Count == 0 ? [c] : c.Subcommands).ToList();
Assert.Equal(7, leaves.Count);
Assert.Equal(8, leaves.Count); // + verify-heads (ScadaBridge#107)
Assert.All(leaves, sub =>
Assert.True(sub.Action != null, $"Leaf command '{sub.Name}' has no action."));
}
@@ -0,0 +1,158 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.CLI.Commands;
namespace ZB.MOM.WW.ScadaBridge.CLI.Tests.Commands;
/// <summary>
/// Tests for <c>scadabridge audit verify-heads</c> (ScadaBridge#107 item 1): reading off-box
/// exported chain heads (file-sink lines and syslog-collector lines), collapsing duplicates,
/// sending <c>VerifyAuditChainHeads</c>, and mapping the verdict to the exit code.
/// </summary>
[Collection("Console")]
public sealed class AuditVerifyHeadsCommandTests : IDisposable
{
private readonly DirectoryInfo _dir = Directory.CreateTempSubdirectory("verify-heads-");
/// <inheritdoc />
public void Dispose() => _dir.Delete(recursive: true);
private static string Line(string month, long seq, char hash, string exportedAt = "2026-05-02T00:00:00Z", string node = "central-a") =>
"{\"v\":1,\"month\":\"" + month + "\",\"lastSeq\":" + seq + ",\"lastHash\":\"" + new string(hash, 64)
+ "\",\"purgedThroughSeq\":null,\"purgedThroughHash\":null,\"chainedRows\":" + seq + ",\"tombstones\":0,"
+ "\"headUpdatedAtUtc\":\"2026-05-02T00:00:00Z\",\"exportedAtUtc\":\"" + exportedAt + "\",\"node\":\"" + node + "\"}";
/// <summary>
/// File-sink lines and RFC 5424 syslog lines both parse; junk is skipped; the same head
/// exported by both nodes (and re-exported) collapses to one record.
/// </summary>
[Fact]
public void ReadRecords_ParsesFileAndSyslogLines_SkipsJunk_AndCollapsesDuplicates()
{
Directory.CreateDirectory(Path.Combine(_dir.FullName, "2026"));
File.WriteAllLines(Path.Combine(_dir.FullName, "2026", "audit-chain-heads-2026-05-02-central-a.jsonl"),
[
Line("2026-05", 10, 'a'),
Line("2026-05", 12, 'b'),
"not json at all",
"",
]);
File.WriteAllLines(Path.Combine(_dir.FullName, "syslog.log"),
[
"Nov 10 03:00:00 collector <109>1 2026-05-02T00:00:00.000Z central-b scadabridge 42 chainhead - "
+ Line("2026-05", 12, 'b', "2026-05-02T01:00:00Z", "central-b"),
"<109>1 2026-05-02T00:00:00.000Z central-b scadabridge 42 chainhead - " + Line("2026-04", 7, 'c'),
]);
File.WriteAllText(Path.Combine(_dir.FullName, "ignored.csv"), Line("2026-03", 1, 'd'));
var read = AuditVerifyHeadsHelpers.ReadRecords(_dir.FullName);
Assert.Equal(2, read.Files);
Assert.Equal(4, read.LinesRead);
Assert.Equal(1, read.LinesSkipped);
Assert.Equal(["2026-04/7", "2026-05/10", "2026-05/12"], read.Records.Select(r => $"{r.Month}/{r.LastSeq}"));
// The duplicate kept is the latest export.
Assert.Equal("central-b", read.Records.Single(r => r.LastSeq == 12).Node);
}
/// <summary>A record missing its hash or sequence is not a record.</summary>
/// <param name="line">The line.</param>
[Theory]
[InlineData("{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":0,\"lastHash\":\"aa\"}")]
[InlineData("{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":3}")]
[InlineData("{\"month\":\"2026-05\",\"lastSeq\":3,\"lastHash\":\"aa\"}")]
[InlineData("{ broken")]
public void TryParseLine_RejectsIncompleteRecords(string line)
{
Assert.Null(AuditVerifyHeadsHelpers.TryParseLine(line));
}
/// <summary>
/// Round trip: the request carries <c>VerifyAuditChainHeads</c> with the distinct records,
/// and the verdict drives the exit code (0 verified, 3 not).
/// </summary>
/// <param name="verified">Whether the fake server reports verified.</param>
/// <param name="format">Output format.</param>
/// <param name="expectedExit">Expected exit code.</param>
/// <param name="expectedText">Text the report must contain.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(true, "table", 0, "Verified : yes")]
[InlineData(false, "table", 3, "Mismatch")]
[InlineData(false, "json", 3, "\"verified\":false")]
public async Task VerifyHeads_SendsTheRecords_AndMapsVerdictToExitCode(bool verified, string format, int expectedExit, string expectedText)
{
var file = Path.Combine(_dir.FullName, "heads.jsonl");
File.WriteAllLines(file, [Line("2026-05", 12, 'b'), Line("2026-05", 12, 'b', "2026-05-02T02:00:00Z")]);
var body = "{\"verified\":" + (verified ? "true" : "false") + ",\"recordsChecked\":1,\"checks\":[{\"month\":\"2026-05\","
+ "\"exportedSeq\":12,\"exportedAtUtc\":\"2026-05-02T02:00:00Z\",\"node\":\"central-a\",\"status\":\""
+ (verified ? "Matches" : "Mismatch") + "\",\"databaseSeq\":12,\"detail\":\"d\"}]}";
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
try
{
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
string? requestBody = null;
var server = Task.Run(async () =>
{
using var client = await listener.AcceptTcpClientAsync();
await using var stream = client.GetStream();
using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
var length = 0;
while (await reader.ReadLineAsync() is { Length: > 0 } line)
if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase))
length = int.Parse(line[15..].Trim());
var buffer = new char[length];
var read = 0;
while (read < length)
{
var count = await reader.ReadAsync(buffer.AsMemory(read));
if (count == 0) throw new EndOfStreamException();
read += count;
}
requestBody = new string(buffer);
var bytes = Encoding.UTF8.GetBytes(body);
var headers = Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n");
await stream.WriteAsync(headers);
await stream.WriteAsync(bytes);
});
var root = AuditCommandTestHarness.BuildRoot();
var (exit, output, _) = AuditCommandTestHarness.Invoke(
root, "audit", "verify-heads", "--path", file,
"--url", $"http://127.0.0.1:{port}", "--username", "u", "--password", "p", "--format", format);
await server.WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal(expectedExit, exit);
Assert.Contains(expectedText, output);
using var payload = JsonDocument.Parse(requestBody!);
Assert.Equal("VerifyAuditChainHeads", payload.RootElement.GetProperty("command").GetString());
var records = payload.RootElement.GetProperty("payload").EnumerateObject()
.First(p => string.Equals(p.Name, "records", StringComparison.OrdinalIgnoreCase)).Value;
Assert.Equal(1, records.GetArrayLength());
Assert.Equal(new string('b', 64), records[0].EnumerateObject()
.First(p => string.Equals(p.Name, "lastHash", StringComparison.OrdinalIgnoreCase)).Value.GetString());
}
finally
{
listener.Stop();
}
}
/// <summary>No records found is an input error, and nothing is sent.</summary>
[Fact]
public void VerifyHeads_NoRecords_ExitsOneWithoutContactingTheServer()
{
var file = Path.Combine(_dir.FullName, "empty.jsonl");
File.WriteAllText(file, "nothing here\n");
var root = AuditCommandTestHarness.BuildRoot();
var (exit, _, _) = AuditCommandTestHarness.Invoke(
root, "audit", "verify-heads", "--path", file, "--url", "http://127.0.0.1:1", "--username", "u", "--password", "p");
Assert.Equal(1, exit);
}
}
@@ -45,7 +45,7 @@ public class ManagementSurfaceParityTests
}
private static readonly HashSet<string> AuditManagementLeaves =
["audit verify-chain", "audit archive list", "audit archive verify"];
["audit verify-chain", "audit verify-heads", "audit archive list", "audit archive verify"];
private static async Task<ConcurrentDictionary<string, string>> ProbeProductionTree(string? omitGroup = null)
{
@@ -56,6 +56,12 @@ public class ManagementSurfaceParityTests
await File.WriteAllTextAsync(file, "{}");
var csv = Path.Combine(directory.FullName, "native.csv");
await File.WriteAllTextAsync(csv, "SourceName,Connection,SourceReference,Filter\nSource,Connection,Path,\n");
// `audit verify-heads` (ScadaBridge#107) reads exported chain heads before it sends anything.
HeadsFile = Path.Combine(directory.FullName, "heads.jsonl");
await File.WriteAllTextAsync(HeadsFile,
"{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":3,\"lastHash\":\"" + new string('a', 64)
+ "\",\"purgedThroughSeq\":null,\"purgedThroughHash\":null,\"chainedRows\":3,\"tombstones\":0,"
+ "\"headUpdatedAtUtc\":\"2026-05-02T00:00:00Z\",\"exportedAtUtc\":\"2026-05-02T00:00:00Z\",\"node\":\"central-a\"}\n");
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
@@ -96,7 +102,8 @@ public class ManagementSurfaceParityTests
foreach (var (leaf, path) in Leaves(root, []))
{
// Separate REST and persistent SignalR surfaces, not management contracts.
// `audit verify-chain` (VerifyAuditChain, ScadaBridge#41) and `audit archive
// `audit verify-chain` (VerifyAuditChain, ScadaBridge#41), `audit verify-heads`
// (VerifyAuditChainHeads, ScadaBridge#107) and `audit archive
// list|verify` (ListAuditArchives / VerifyAuditArchive, ScadaBridge#42) are the
// audit leaves that ARE management contracts, so they stay in the probe.
if (path.Length == 0 || (path[0] == "audit" && !AuditManagementLeaves.Contains(string.Join(' ', path)))
@@ -135,8 +142,11 @@ public class ManagementSurfaceParityTests
foreach (var entry in Leaves(child, [.. path, child.Name])) yield return entry;
}
private static string HeadsFile = "";
private static string ValueFor(Option option, string path, string file, string csv)
{
if (path == "audit verify-heads" && option.Name == "--path") return HeadsFile;
var type = option.GetType().GetGenericArguments().FirstOrDefault();
type = Nullable.GetUnderlyingType(type ?? typeof(string)) ?? type;
if (type == typeof(int) || type == typeof(long) || type == typeof(double)) return "1";
@@ -0,0 +1,81 @@
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// Pure verdict table of <see cref="AuditChainHeadComparer"/> (ScadaBridge#107 item 1) for the
/// branches the SQL Server tests do not reach: a link carried by a tombstone, the partition
/// anchor itself, and malformed input.
/// </summary>
public sealed class AuditChainHeadComparerTests
{
private static readonly byte[] H1 = Enumerable.Repeat((byte)1, 32).ToArray();
private static readonly byte[] H2 = Enumerable.Repeat((byte)2, 32).ToArray();
private static AuditChainHeadRecord Exported(long seq, byte[] hash) => new(
1, "2026-05", seq, AuditChainHeadComparer.ToHex(hash), null, null, seq, 0, DateTime.UtcNow, DateTime.UtcNow, "central-a");
private static AuditChainHeadSnapshot Head(long seq, byte[] hash, long? anchor = null, byte[]? anchorHash = null) =>
new(new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc), seq, hash, anchor, anchorHash, seq, 0, DateTime.UtcNow);
private static Func<long, Task<AuditChainHeadComparer.StoredLink>> Link(byte[]? row, byte[]? tombstone)
=> _ => Task.FromResult(new AuditChainHeadComparer.StoredLink(row, tombstone));
private static Func<long, Task<AuditChainHeadComparer.StoredLink>> NoLookup
=> _ => throw new InvalidOperationException("no link lookup expected");
/// <summary>The exported link was later purged per channel: the tombstone carries the hash.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task GrownChain_LinkOnATombstone_IsExtended_OrMismatch()
{
Assert.Equal(AuditChainHeadStatus.Extended,
(await AuditChainHeadComparer.CompareAsync(Exported(3, H1), Head(9, H2), Link(null, H1))).Status);
Assert.Equal(AuditChainHeadStatus.Mismatch,
(await AuditChainHeadComparer.CompareAsync(Exported(3, H1), Head(9, H2), Link(null, H2))).Status);
}
/// <summary>The exported head is the partition-purge anchor: compared with the anchor's hash, no lookup.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task GrownChain_ExportedHeadIsTheAnchor_ComparesTheAnchorHash()
{
Assert.Equal(AuditChainHeadStatus.Extended,
(await AuditChainHeadComparer.CompareAsync(Exported(4, H1), Head(9, H2, 4, H1), NoLookup)).Status);
Assert.Equal(AuditChainHeadStatus.Mismatch,
(await AuditChainHeadComparer.CompareAsync(Exported(4, H1), Head(9, H2, 4, H2), NoLookup)).Status);
Assert.Equal(AuditChainHeadStatus.Purged,
(await AuditChainHeadComparer.CompareAsync(Exported(2, H1), Head(9, H2, 4, H2), NoLookup)).Status);
}
/// <summary>A record whose hash is not 32 bytes of hex can never match.</summary>
/// <param name="hash">The exported hash text.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData("abc")]
[InlineData("zz0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f")]
public async Task MalformedExportedHash_IsMismatch(string hash)
{
var record = Exported(3, H1) with { LastHash = hash };
Assert.Equal(AuditChainHeadStatus.Mismatch,
(await AuditChainHeadComparer.CompareAsync(record, Head(3, H1), NoLookup)).Status);
}
/// <summary>Only the tamper verdicts count as failures.</summary>
/// <param name="status">The verdict.</param>
/// <param name="failure">Whether it is a failure.</param>
[Theory]
[InlineData(AuditChainHeadStatus.Matches, false)]
[InlineData(AuditChainHeadStatus.Extended, false)]
[InlineData(AuditChainHeadStatus.Purged, false)]
[InlineData(AuditChainHeadStatus.Mismatch, true)]
[InlineData(AuditChainHeadStatus.Regressed, true)]
[InlineData(AuditChainHeadStatus.MissingHead, true)]
[InlineData(AuditChainHeadStatus.MissingLink, true)]
[InlineData(AuditChainHeadStatus.CountMismatch, true)]
public void IsFailure_IsTrueOnlyForTamperVerdicts(AuditChainHeadStatus status, bool failure)
{
Assert.Equal(failure, new AuditChainHeadCheck("2026-05", 1, DateTime.UtcNow, null, status, 1, "").IsFailure);
}
}
@@ -0,0 +1,312 @@
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// SQL Server tests for the chain-head export store (ScadaBridge#107 item 1): heads read with
/// their counts, and exported heads checked against the database. The key case is the forgery
/// the export exists for: rows rewritten AND every later link and the head recomputed — which
/// <c>verify-chain</c> accepts — is caught because the exported hash no longer matches.
/// </summary>
/// <remarks>Each test owns one month (partition) of the shared fixture database.</remarks>
public sealed class AuditChainHeadStoreTests : IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
/// <summary>Creates the test class with the shared migrated SQL Server fixture.</summary>
/// <param name="fixture">The shared migration fixture.</param>
public AuditChainHeadStoreTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
/// <summary>The head snapshot carries the chain end and the live row + tombstone counts.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ReadHeads_ReportsTheChainEnd_AndRowAndTombstoneCounts()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 3);
await InsertAsync(Enumerable.Range(1, 5).Select(i => NewEvent(month.AddDays(i))));
await InsertAsync([NewEvent(month.AddHours(1), AuditChannel.DbOutbound)]);
await DrainAsync();
await using (var ctx = CreateContext())
{
Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100));
}
var head = await HeadAsync(month);
Assert.Equal(6, head.LastSeq);
Assert.Equal(5, head.ChainedRows);
Assert.Equal(1, head.Tombstones);
Assert.Null(head.PurgedThroughSeq);
}
/// <summary>
/// Unchanged chain: Matches. The chain grew: Extended. Both centrals' duplicate exports
/// collapse to one check.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ExportedHeads_MatchTheUnchangedChain_AndAreExtendedByLaterLinks()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 4);
await InsertAsync(Enumerable.Range(1, 4).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var exported = Export(await HeadAsync(month));
var same = await VerifyAsync(exported, exported with { Node = "central-b" });
Assert.True(same.Verified);
Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single(same.Checks).Status);
await InsertAsync([NewEvent(month.AddDays(10))]);
await DrainAsync();
var grown = await VerifyAsync(exported);
Assert.True(grown.Verified);
var check = Assert.Single(grown.Checks);
Assert.Equal(AuditChainHeadStatus.Extended, check.Status);
Assert.Equal(5, check.DatabaseSeq);
}
/// <summary>
/// The forgery: a chained row's content is rewritten and every link from it plus the head
/// are recomputed with the real hasher. <c>VerifyMonthAsync</c> reports the forged chain
/// Intact (positive control: the in-database chain alone cannot see it); the exported head
/// does not match it.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ARewrittenAndRehashedChain_PassesVerifyChain_ButFailsTheExportedHead()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 5);
await InsertAsync(Enumerable.Range(1, 6).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var exported = Export(await HeadAsync(month));
await ForgeChainAsync(month, fromSeq: 3);
await using (var ctx = CreateContext())
{
var chain = await new AuditLogHashChain(ctx).VerifyMonthAsync(month);
Assert.Equal(AuditChainStatus.Intact, chain.Status);
}
var result = await VerifyAsync(exported);
Assert.False(result.Verified);
Assert.Equal(AuditChainHeadStatus.Mismatch, Assert.Single(result.Checks).Status);
}
/// <summary>Each structural tampering maps to its own verdict.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task TamperedHeadsAndRows_AreReported_WithTheirOwnVerdicts()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Head rolled back below the export (tail deletion + head rewrite).
var regressed = Month(2026, 6);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(regressed.AddDays(i))));
await DrainAsync();
var regressedExport = Export(await HeadAsync(regressed));
await ExecAsync("UPDATE dbo.AuditLogChainHead SET LastSeq = LastSeq - 1 WHERE ChainMonth = @m;", regressed);
// Head row deleted.
var missingHead = Month(2026, 7);
await InsertAsync([NewEvent(missingHead.AddDays(1))]);
await DrainAsync();
var missingHeadExport = Export(await HeadAsync(missingHead));
await ExecAsync("DELETE FROM dbo.AuditLogChainHead WHERE ChainMonth = @m;", missingHead);
// A chained row deleted with no tombstone, head unchanged: counts no longer add up.
var counted = Month(2026, 8);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(counted.AddDays(i))));
await DrainAsync();
var countedExport = Export(await HeadAsync(counted));
await ExecAsync("DELETE FROM dbo.AuditLog WHERE ChainSeq = 2 AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", counted);
// The exported link itself deleted after the chain grew past it.
var missingLink = Month(2026, 9);
await InsertAsync(Enumerable.Range(1, 2).Select(i => NewEvent(missingLink.AddDays(i))));
await DrainAsync();
var missingLinkExport = Export(await HeadAsync(missingLink));
await InsertAsync([NewEvent(missingLink.AddDays(5))]);
await DrainAsync();
await ExecAsync("DELETE FROM dbo.AuditLog WHERE ChainSeq = 2 AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", missingLink);
var result = await VerifyAsync(regressedExport, missingHeadExport, countedExport, missingLinkExport);
Assert.False(result.Verified);
var byMonth = result.Checks.ToDictionary(c => c.Month, c => c.Status);
Assert.Equal(AuditChainHeadStatus.Regressed, byMonth["2026-06"]);
Assert.Equal(AuditChainHeadStatus.MissingHead, byMonth["2026-07"]);
Assert.Equal(AuditChainHeadStatus.CountMismatch, byMonth["2026-08"]);
Assert.Equal(AuditChainHeadStatus.MissingLink, byMonth["2026-09"]);
}
/// <summary>
/// Sanctioned purges are not tamper evidence: a per-channel purge (row to tombstone) keeps
/// Matches; after a partition switch the exported head is the anchor, so the check still
/// passes, and an older exported sequence the switch removed is reported Purged.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task SanctionedPurges_AreNotFailures()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 10);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var early = Export(await HeadAsync(month));
await InsertAsync([NewEvent(month.AddDays(5), AuditChannel.Notification)]);
await DrainAsync();
var late = Export(await HeadAsync(month));
await using (var ctx = CreateContext())
{
Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("Notification", month.AddDays(6), 100));
}
Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single((await VerifyAsync(late)).Checks).Status);
await using (var ctx = CreateContext())
{
await new AuditLogRepository(ctx).SwitchOutPartitionAsync(month);
}
var afterSwitch = await VerifyAsync(late, early);
Assert.True(afterSwitch.Verified);
var statuses = afterSwitch.Checks.ToDictionary(c => c.ExportedSeq, c => c.Status);
Assert.Equal(AuditChainHeadStatus.Matches, statuses[4]);
Assert.Equal(AuditChainHeadStatus.Purged, statuses[3]);
}
private static AuditChainHeadRecord Export(AuditChainHeadSnapshot head) => new(
AuditChainHeadRecord.CurrentVersion,
AuditChainHasher.PartitionLabel(head.Month),
head.LastSeq,
AuditChainHeadComparer.ToHex(head.LastHash),
head.PurgedThroughSeq,
head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash),
head.ChainedRows,
head.Tombstones,
head.UpdatedAtUtc,
DateTime.UtcNow,
"central-a");
private async Task<AuditChainHeadSnapshot> HeadAsync(DateTime month)
{
await using var ctx = CreateContext();
var heads = await new AuditLogHashChain(ctx).ReadHeadsAsync();
return Assert.Single(heads, h => h.Month == month);
}
private async Task<AuditChainHeadVerificationResult> VerifyAsync(params AuditChainHeadRecord[] records)
{
await using var ctx = CreateContext();
return await new AuditLogHashChain(ctx).VerifyExportedHeadsAsync(records);
}
/// <summary>
/// Rewrites the row at <paramref name="fromSeq"/> and re-links it and every later row plus the
/// head with the production hasher — what a principal with full write access could do.
/// </summary>
private async Task ForgeChainAsync(DateTime month, long fromSeq)
{
await using var conn = _fixture.OpenConnection();
await using (var edit = new SqlCommand(
"UPDATE dbo.AuditLog SET Target = N'forged' WHERE ChainSeq = @s AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", conn))
{
edit.Parameters.AddWithValue("@s", fromSeq);
edit.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month });
Assert.Equal(1, await edit.ExecuteNonQueryAsync());
}
var rows = new List<AuditChainRow>();
await using (var read = new SqlCommand(
$"SELECT {AuditLogHashChain.RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " +
"WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m) ORDER BY ChainSeq;", conn))
{
read.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month });
await using var reader = await read.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
rows.Add(AuditLogHashChain.ReadRow(reader, withLink: true));
}
}
var previous = rows.Single(r => r.ChainSeq == fromSeq - 1).RowHash!;
foreach (var row in rows.Where(r => r.ChainSeq >= fromSeq))
{
var hash = AuditChainHasher.ComputeRowHash(previous, month, row.ChainSeq!.Value, row);
await using var update = new SqlCommand("UPDATE dbo.AuditLog SET RowHash = @h WHERE EventId = @e AND OccurredAtUtc = @o;", conn);
update.Parameters.AddWithValue("@h", hash);
update.Parameters.AddWithValue("@e", row.EventId);
update.Parameters.Add(new SqlParameter("@o", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = row.OccurredAtUtc });
Assert.Equal(1, await update.ExecuteNonQueryAsync());
previous = hash;
}
await using var head = new SqlCommand("UPDATE dbo.AuditLogChainHead SET LastHash = @h WHERE ChainMonth = @m;", conn);
head.Parameters.AddWithValue("@h", previous);
head.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.Date) { Value = month });
Assert.Equal(1, await head.ExecuteNonQueryAsync());
}
private async Task ExecAsync(string sql, DateTime month)
{
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand(sql, conn);
cmd.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month });
Assert.True(await cmd.ExecuteNonQueryAsync() >= 1);
}
private async Task InsertAsync(IEnumerable<AuditEvent> events)
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
foreach (var e in events)
{
await repo.InsertIfNotExistsAsync(e);
}
}
private async Task DrainAsync()
{
await using var ctx = CreateContext();
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
}
}
throw new InvalidOperationException("Chain did not drain.");
}
private static DateTime Month(int year, int month) => new(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
private ScadaBridgeDbContext CreateContext()
=> new(new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
private static AuditEvent NewEvent(DateTime occurredAtUtc, AuditChannel channel = AuditChannel.ApiOutbound) =>
ScadaBridgeAuditEventFactory.Create(
channel: channel,
kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall,
status: AuditStatus.Delivered,
occurredAtUtc: occurredAtUtc,
actor: "head-test",
target: "erp",
sourceNode: "central-a",
sourceSiteId: "head-test",
correlationId: Guid.NewGuid());
}
@@ -0,0 +1,200 @@
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// The hardened two-login setup (ScadaBridge#107 item 3), end to end on SQL Server: a writer
/// login (<c>db_datareader</c> + <c>db_datawriter</c> + <c>scadabridge_audit_writer</c>) and a
/// maintenance login (<c>scadabridge_audit_chainer</c> + <c>scadabridge_audit_purger</c> only),
/// wired through <c>AddConfigurationDatabase(writer, maintenance)</c>. The writer ingests but
/// cannot chain, update or delete; the maintenance services run on the maintenance login and
/// can chain, verify, export heads, per-channel purge, enumerate partitions, archive and switch.
/// </summary>
public sealed class AuditMaintenanceLoginTests : IClassFixture<MsSqlMigrationFixture>, IAsyncLifetime
{
private const string Password = "Sb_W22_maint#2026";
private readonly MsSqlMigrationFixture _fixture;
private readonly string _suffix = Guid.NewGuid().ToString("N")[..10];
private readonly DirectoryInfo _archiveDir = Directory.CreateTempSubdirectory("maint-login-archive-");
/// <summary>Creates the test class with its own migrated SQL Server database.</summary>
/// <param name="fixture">The per-class migration fixture.</param>
public AuditMaintenanceLoginTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
private string WriterLogin => "sbw_" + _suffix;
private string MaintenanceLogin => "sbm_" + _suffix;
/// <inheritdoc />
public async Task InitializeAsync()
{
if (!_fixture.Available)
{
return;
}
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand($@"
CREATE LOGIN [{WriterLogin}] WITH PASSWORD = N'{Password}', CHECK_POLICY = OFF;
CREATE LOGIN [{MaintenanceLogin}] WITH PASSWORD = N'{Password}', CHECK_POLICY = OFF;
CREATE USER [{WriterLogin}] FOR LOGIN [{WriterLogin}];
CREATE USER [{MaintenanceLogin}] FOR LOGIN [{MaintenanceLogin}];
ALTER ROLE db_datareader ADD MEMBER [{WriterLogin}];
ALTER ROLE db_datawriter ADD MEMBER [{WriterLogin}];
ALTER ROLE scadabridge_audit_writer ADD MEMBER [{WriterLogin}];
ALTER ROLE scadabridge_audit_chainer ADD MEMBER [{MaintenanceLogin}];
ALTER ROLE scadabridge_audit_purger ADD MEMBER [{MaintenanceLogin}];", conn);
await cmd.ExecuteNonQueryAsync();
}
/// <inheritdoc />
public async Task DisposeAsync()
{
_archiveDir.Delete(recursive: true);
if (!_fixture.Available)
{
return;
}
SqlConnection.ClearAllPools();
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand($@"
DECLARE @kill nvarchar(max) = N'';
SELECT @kill = @kill + N'KILL ' + CAST(session_id AS nvarchar(10)) + N';' FROM sys.dm_exec_sessions
WHERE login_name IN (N'{WriterLogin}', N'{MaintenanceLogin}');
EXEC sp_executesql @kill;
IF USER_ID(N'{WriterLogin}') IS NOT NULL DROP USER [{WriterLogin}];
IF USER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP USER [{MaintenanceLogin}];
IF SUSER_ID(N'{WriterLogin}') IS NOT NULL DROP LOGIN [{WriterLogin}];
IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];", conn);
await cmd.ExecuteNonQueryAsync();
}
/// <summary>
/// Writer ingests and is refused every maintenance write; the maintenance services, resolved
/// from DI on the separate login, do the whole chain + purge + archive cycle.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task HardenedTwoLoginSetup_WriterIngestsOnly_MaintenanceLoginRunsChainAndPurges()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var writerCs = As(WriterLogin);
var maintenanceCs = As(MaintenanceLogin);
var month = new DateTime(2027, 2, 1, 0, 0, 0, DateTimeKind.Utc);
// The writer login ingests.
await using (var writerCtx = Context(writerCs))
{
var repo = new AuditLogRepository(writerCtx);
foreach (var i in Enumerable.Range(1, 4))
{
await repo.InsertIfNotExistsAsync(NewEvent(month.AddDays(i), AuditChannel.ApiOutbound));
}
await repo.InsertIfNotExistsAsync(NewEvent(month.AddHours(2), AuditChannel.DbOutbound));
}
// Negative controls: the writer login can neither chain nor delete.
await using (var writerCtx = Context(writerCs))
{
await Assert.ThrowsAsync<SqlException>(() => new AuditLogHashChain(writerCtx).ChainPendingAsync(100));
await Assert.ThrowsAsync<SqlException>(() =>
new AuditLogRepository(writerCtx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100));
}
var services = new ServiceCollection();
services.AddLogging();
services.AddConfigurationDatabase(writerCs, maintenanceCs);
services.AddSingleton(Options.Create(new AuditLogArchiveOptions
{
Enabled = true,
Directory = _archiveDir.FullName,
RowGroupSize = 1_000,
}));
await using var provider = services.BuildServiceProvider();
await using (var scope = provider.CreateAsyncScope())
{
var maintenance = scope.ServiceProvider.GetRequiredService<AuditMaintenanceContext>();
Assert.True(maintenance.UsesSeparateLogin);
Assert.Equal(MaintenanceLogin, await LoginNameAsync(maintenance.Context));
Assert.Equal(WriterLogin, await LoginNameAsync(scope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>()));
var chain = scope.ServiceProvider.GetRequiredService<IAuditLogHashChain>();
Assert.Equal(5, (await chain.ChainPendingAsync(100)).RowsChained);
Assert.Equal(AuditChainStatus.Intact, (await chain.VerifyMonthAsync(month)).Status);
var head = Assert.Single(await scope.ServiceProvider.GetRequiredService<IAuditChainHeadStore>().ReadHeadsAsync());
Assert.Equal(5, head.LastSeq);
var repo = scope.ServiceProvider.GetRequiredKeyedService<IAuditLogRepository>(AuditMaintenanceContext.ServiceKey);
Assert.Equal(1, await repo.PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100));
Assert.Contains(month, await repo.GetPartitionBoundariesOlderThanAsync(month.AddYears(5)));
var archive = scope.ServiceProvider.GetRequiredKeyedService<IAuditLogArchive>(AuditMaintenanceContext.ServiceKey);
var manifest = await archive.ArchivePartitionAsync(month);
Assert.Equal(4, manifest.RowCount);
Assert.Equal(4, await archive.PurgeArchivedPartitionAsync(manifest));
Assert.Equal(AuditChainStatus.Intact, (await chain.VerifyMonthAsync(month)).Status);
}
// Unset maintenance string: the maintenance context is the runtime context, as before.
var plain = new ServiceCollection();
plain.AddLogging();
plain.AddConfigurationDatabase(writerCs);
await using var plainProvider = plain.BuildServiceProvider();
await using var plainScope = plainProvider.CreateAsyncScope();
var shared = plainScope.ServiceProvider.GetRequiredService<AuditMaintenanceContext>();
Assert.False(shared.UsesSeparateLogin);
Assert.Same(plainScope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>(), shared.Context);
}
private string As(string login) => new SqlConnectionStringBuilder(_fixture.ConnectionString)
{
UserID = login,
Password = Password,
IntegratedSecurity = false,
}.ConnectionString;
private static ScadaBridgeDbContext Context(string cs)
=> new(new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(cs).Options);
private static async Task<string> LoginNameAsync(ScadaBridgeDbContext ctx)
{
var conn = ctx.Database.GetDbConnection();
await conn.OpenAsync();
try
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT SUSER_SNAME();";
return (string)(await cmd.ExecuteScalarAsync())!;
}
finally
{
await conn.CloseAsync();
}
}
private static AuditEvent NewEvent(DateTime occurredAtUtc, AuditChannel channel) =>
ScadaBridgeAuditEventFactory.Create(
channel: channel,
kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall,
status: AuditStatus.Delivered,
occurredAtUtc: occurredAtUtc,
actor: "maint-login-test",
target: "erp",
sourceNode: "central-a",
sourceSiteId: "maint-login-test",
correlationId: Guid.NewGuid());
}
@@ -171,6 +171,8 @@ public class RequiredRoleMatrixTests
// AuditLog hash-chain verification (ScadaBridge#41) carries audit query's
// OperationalAudit permission (alog.md §15.1); written out by hand to keep the freeze.
["VerifyAuditChain"] = [Roles.Administrator, Roles.Viewer],
// Off-box exported chain heads (ScadaBridge#107): same audit-read gate.
["VerifyAuditChainHeads"] = [Roles.Administrator, Roles.Viewer],
// AuditLog Parquet archives (ScadaBridge#42): same audit-read gate as verify-chain.
["ListAuditArchives"] = [Roles.Administrator, Roles.Viewer],
["VerifyAuditArchive"] = [Roles.Administrator, Roles.Viewer],
@@ -0,0 +1,110 @@
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
/// <summary>
/// The <c>VerifyAuditChainHeads</c> management command (ScadaBridge#107 item 1): role gate,
/// site-scope refusal, input bounds, and the verdicts reaching the wire with enums as names.
/// </summary>
public class VerifyAuditChainHeadsCommandTests : TestKit
{
private readonly IAuditChainHeadStore _store = Substitute.For<IAuditChainHeadStore>();
private static readonly AuditChainHeadRecord Record = new(
1, "2026-05", 12, new string('a', 64), null, null, 12, 0,
new DateTime(2026, 5, 3, 0, 0, 0, DateTimeKind.Utc), new DateTime(2026, 5, 3, 1, 0, 0, DateTimeKind.Utc), "central-a");
private IActorRef CreateActor()
{
var services = new ServiceCollection().AddTestConfigurationTransaction();
services.AddScoped(_ => _store);
return Sys.ActorOf(Props.Create(() => new ManagementActor(
services.BuildServiceProvider(), NullLogger<ManagementActor>.Instance)));
}
private static ManagementEnvelope Envelope(object command, string[] sites, params string[] roles) =>
new(new AuthenticatedUser("u", "U", roles, sites), command, Guid.NewGuid().ToString("N"));
/// <summary>A Viewer gets the store's verdicts; the records reach the store unchanged.</summary>
[Fact]
public void Viewer_GetsTheVerdicts_ForTheSuppliedRecords()
{
var check = new AuditChainHeadCheck("2026-05", 12, Record.ExportedAtUtc, "central-a",
AuditChainHeadStatus.Mismatch, 12, "rewritten");
_store.VerifyExportedHeadsAsync(Arg.Any<IReadOnlyList<AuditChainHeadRecord>>(), Arg.Any<CancellationToken>())
.Returns(new AuditChainHeadVerificationResult(false, 1, [check]));
CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), [], "Viewer"));
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
using var json = JsonDocument.Parse(response.JsonData);
Assert.False(json.RootElement.GetProperty("verified").GetBoolean());
Assert.Equal("Mismatch", json.RootElement.GetProperty("checks")[0].GetProperty("status").GetString());
_store.Received(1).VerifyExportedHeadsAsync(
Arg.Is<IReadOnlyList<AuditChainHeadRecord>>(r => r.Count == 1 && r[0] == Record), Arg.Any<CancellationToken>());
}
/// <summary>A Designer/Deployer without an OperationalAudit role is refused.</summary>
[Fact]
public void NonAuditRole_IsUnauthorized()
{
CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), [], "Designer", "Deployer"));
ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
_store.DidNotReceiveWithAnyArgs().VerifyExportedHeadsAsync(default!, default);
}
/// <summary>A site-scoped Viewer is refused: the chain spans every site.</summary>
[Fact]
public void SiteScopedViewer_IsRefused()
{
CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), ["3"], "Viewer"));
var response = ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
Assert.Contains("spans every site", response.Message);
_store.DidNotReceiveWithAnyArgs().VerifyExportedHeadsAsync(default!, default);
}
/// <summary>No records, or more than the cap, is a caller error.</summary>
/// <param name="count">Records supplied.</param>
[Theory]
[InlineData(0)]
[InlineData(ManagementActor.MaxChainHeadRecords + 1)]
public void EmptyOrOversizedInput_IsInvalidArgument(int count)
{
CreateActor().Tell(Envelope(
new VerifyAuditChainHeadsCommand(Enumerable.Repeat(Record, count).ToList()), [], "Administrator"));
var response = ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5));
Assert.Equal("INVALID_ARGUMENT", response.ErrorCode);
_store.DidNotReceiveWithAnyArgs().VerifyExportedHeadsAsync(default!, default);
}
/// <summary>A chain lock that cannot be had surfaces as CHAIN_BUSY.</summary>
[Fact]
public void BusyChain_IsChainBusy()
{
_store.VerifyExportedHeadsAsync(Arg.Any<IReadOnlyList<AuditChainHeadRecord>>(), Arg.Any<CancellationToken>())
.Returns<Task<AuditChainHeadVerificationResult>>(_ => throw new TimeoutException("busy"));
CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), [], "Administrator"));
Assert.Equal("CHAIN_BUSY", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
}
/// <summary>Runs under the long-running Ask timeout.</summary>
[Fact]
public void VerifyAuditChainHeads_UsesLongRunningAskTimeout()
{
Assert.Equal(TimeSpan.FromMinutes(5),
ManagementEndpoints.ResolveAskTimeout(new ManagementServiceOptions(), typeof(VerifyAuditChainHeadsCommand)));
}
}