533414a16d
#113: AuditLogPartitionMaintenance (monthly SPLIT RANGE) and the backfill-source-node endpoint now use the AuditLog:MaintenanceConnectionString login when set (unset = unchanged). Migration AuditMaintenanceGrants gives scadabridge_audit_purger ALTER ANY DATASPACE and column-level UPDATE (SourceNode). #114: archive cleanup deletes the writer's own *.tmp files and manifest-less data files older than 24 h inside {Directory}/{YYYY} (DeletedOrphan, dry-run aware). #115: head-export failures bump scadabridge.audit.chain_head_export.failures and Degrade the /health/ready entry audit-chain-head-export (failing or stale > 2 intervals). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
208 lines
9.3 KiB
C#
208 lines
9.3 KiB
C#
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|
using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central.HeadExport;
|
|
|
|
/// <summary>
|
|
/// <see cref="AuditChainHeadExportService"/> (ScadaBridge#107 item 1): first tick exports every
|
|
/// head, later ticks only the changed ones, a failed write is retried with the same heads, and
|
|
/// the options validator guards each sink's settings.
|
|
/// </summary>
|
|
public sealed class AuditChainHeadExportServiceTests
|
|
{
|
|
private sealed class FakeStore : IAuditChainHeadStore
|
|
{
|
|
public List<AuditChainHeadSnapshot> Heads { get; } = [];
|
|
|
|
public Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsAsync(CancellationToken ct = default)
|
|
=> Task.FromResult<IReadOnlyList<AuditChainHeadSnapshot>>(Heads.ToList());
|
|
|
|
public Task<AuditChainHeadVerificationResult> VerifyExportedHeadsAsync(
|
|
IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct = default) => throw new NotSupportedException();
|
|
}
|
|
|
|
private sealed class RecordingSink : IAuditChainHeadSink
|
|
{
|
|
public List<IReadOnlyList<AuditChainHeadRecord>> Writes { get; } = [];
|
|
|
|
public bool Fail { get; set; }
|
|
|
|
public string Describe => "recording";
|
|
|
|
public Task WriteAsync(IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct)
|
|
{
|
|
if (Fail)
|
|
{
|
|
throw new IOException("share offline");
|
|
}
|
|
|
|
Writes.Add(records);
|
|
return Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
private static AuditChainHeadSnapshot Head(int month, long seq, long rows) => new(
|
|
new DateTime(2026, month, 1, 0, 0, 0, DateTimeKind.Utc), seq, Enumerable.Repeat((byte)seq, 32).ToArray(),
|
|
null, null, rows, 0, new DateTime(2026, month, 2, 0, 0, 0, DateTimeKind.Utc));
|
|
|
|
private static (AuditChainHeadExportService Service, FakeStore Store, RecordingSink Sink) Create(
|
|
bool exportUnchanged = false, AuditChainHeadExportStatus? status = null, TimeProvider? time = null)
|
|
{
|
|
var store = new FakeStore();
|
|
var sink = new RecordingSink();
|
|
var services = new ServiceCollection();
|
|
services.AddScoped<IAuditChainHeadStore>(_ => store);
|
|
var options = Options.Create(new AuditLogHashChainOptions
|
|
{
|
|
HeadExport = new AuditChainHeadExportOptions { Enabled = true, ExportUnchanged = exportUnchanged },
|
|
});
|
|
var service = new AuditChainHeadExportService(
|
|
services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>(),
|
|
options, sink, "central-a", NullLogger<AuditChainHeadExportService>.Instance, time, status);
|
|
return (service, store, sink);
|
|
}
|
|
|
|
/// <summary>First tick: every head; next tick: only the head that moved; unchanged: nothing.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Tick_ExportsAllHeadsFirst_ThenOnlyChangedOnes()
|
|
{
|
|
var (service, store, sink) = Create();
|
|
store.Heads.AddRange([Head(4, 10, 10), Head(5, 20, 20)]);
|
|
|
|
Assert.Equal(2, await service.TickAsync(default));
|
|
Assert.Equal(0, await service.TickAsync(default));
|
|
|
|
store.Heads[1] = Head(5, 25, 25);
|
|
Assert.Equal(1, await service.TickAsync(default));
|
|
|
|
// A per-channel purge (row -> tombstone) keeps rows + tombstones, so it is not a change;
|
|
// a row disappearing without a tombstone is.
|
|
store.Heads[0] = Head(4, 10, 9);
|
|
Assert.Equal(1, await service.TickAsync(default));
|
|
|
|
Assert.Equal(3, sink.Writes.Count);
|
|
Assert.Equal(["2026-04", "2026-05"], sink.Writes[0].Select(r => r.Month));
|
|
var moved = Assert.Single(sink.Writes[1]);
|
|
Assert.Equal(("2026-05", 25L, string.Concat(Enumerable.Repeat("19", 32))), (moved.Month, moved.LastSeq, moved.LastHash));
|
|
Assert.Equal("central-a", moved.Node);
|
|
Assert.Equal(AuditChainHeadRecord.CurrentVersion, moved.V);
|
|
}
|
|
|
|
/// <summary>ExportUnchanged re-exports every head every tick.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Tick_WithExportUnchanged_ExportsEveryHeadEveryTick()
|
|
{
|
|
var (service, store, _) = Create(exportUnchanged: true);
|
|
store.Heads.AddRange([Head(4, 10, 10), Head(5, 20, 20)]);
|
|
|
|
Assert.Equal(2, await service.TickAsync(default));
|
|
Assert.Equal(2, await service.TickAsync(default));
|
|
}
|
|
|
|
/// <summary>A failed write is contained and the same heads go out on the next tick.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Tick_AfterAFailedWrite_RetriesTheSameHeads()
|
|
{
|
|
var (service, store, sink) = Create();
|
|
store.Heads.Add(Head(4, 10, 10));
|
|
sink.Fail = true;
|
|
|
|
Assert.Equal(0, await service.TickAsync(default));
|
|
sink.Fail = false;
|
|
Assert.Equal(1, await service.TickAsync(default));
|
|
Assert.Equal(10, Assert.Single(Assert.Single(sink.Writes)).LastSeq);
|
|
}
|
|
|
|
/// <summary>
|
|
/// ScadaBridge#115: a failed tick bumps the failure counter metric and the status
|
|
/// (consecutive + total + last error); the next success clears the consecutive count.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task FailedTick_RecordsMetricAndStatus_SuccessResetsConsecutive()
|
|
{
|
|
var status = new AuditChainHeadExportStatus();
|
|
var (service, store, sink) = Create(status: status);
|
|
store.Heads.Add(Head(4, 10, 10));
|
|
long measured = 0;
|
|
using var listener = new System.Diagnostics.Metrics.MeterListener();
|
|
listener.InstrumentPublished = (i, l) =>
|
|
{
|
|
if (i.Name == "scadabridge.audit.chain_head_export.failures")
|
|
{
|
|
l.EnableMeasurementEvents(i);
|
|
}
|
|
};
|
|
listener.SetMeasurementEventCallback<long>((_, v, _, _) => Interlocked.Add(ref measured, v));
|
|
listener.Start();
|
|
|
|
sink.Fail = true;
|
|
await service.TickAsync(default);
|
|
await service.TickAsync(default);
|
|
|
|
Assert.Equal(2, Interlocked.Read(ref measured));
|
|
Assert.Equal(2, status.ConsecutiveFailures);
|
|
Assert.Equal(2, status.TotalFailures);
|
|
Assert.Equal("share offline", status.LastError);
|
|
Assert.Null(status.LastSuccessUtc);
|
|
|
|
sink.Fail = false;
|
|
await service.TickAsync(default);
|
|
|
|
Assert.Equal(0, status.ConsecutiveFailures);
|
|
Assert.Equal(2, status.TotalFailures);
|
|
Assert.NotNull(status.LastSuccessUtc);
|
|
Assert.Equal(2, Interlocked.Read(ref measured));
|
|
}
|
|
|
|
/// <summary>The validator ignores a disabled export and checks each sink's settings when enabled.</summary>
|
|
/// <param name="configure">Which settings to apply.</param>
|
|
/// <param name="expectedError">A fragment of the expected failure, or null when valid.</param>
|
|
[Theory]
|
|
[InlineData("disabled-bad", null)]
|
|
[InlineData("file-ok", null)]
|
|
[InlineData("file-relative", "File:Directory")]
|
|
[InlineData("file-missing", "File:Directory")]
|
|
[InlineData("syslog-ok", null)]
|
|
[InlineData("syslog-no-host", "Syslog:Host")]
|
|
[InlineData("syslog-bad-port", "Syslog:Port")]
|
|
[InlineData("syslog-bad-facility", "Syslog:Facility")]
|
|
[InlineData("interval-zero", "IntervalMinutes")]
|
|
public void Validator_ChecksTheSelectedSink(string configure, string? expectedError)
|
|
{
|
|
var root = OperatingSystem.IsWindows() ? @"C:\heads" : "/var/heads";
|
|
var export = configure switch
|
|
{
|
|
"disabled-bad" => new AuditChainHeadExportOptions { Enabled = false, IntervalMinutes = 0 },
|
|
"file-ok" => new AuditChainHeadExportOptions { Enabled = true, File = { Directory = root } },
|
|
"file-relative" => new AuditChainHeadExportOptions { Enabled = true, File = { Directory = "heads" } },
|
|
"file-missing" => new AuditChainHeadExportOptions { Enabled = true },
|
|
"syslog-ok" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "collector" } },
|
|
"syslog-no-host" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog },
|
|
"syslog-bad-port" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "c", Port = 0 } },
|
|
"syslog-bad-facility" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "c", Facility = 24 } },
|
|
_ => new AuditChainHeadExportOptions { Enabled = true, IntervalMinutes = 0, File = { Directory = root } },
|
|
};
|
|
|
|
var result = new AuditLogHashChainOptionsValidator().Validate(null, new AuditLogHashChainOptions { HeadExport = export });
|
|
|
|
if (expectedError is null)
|
|
{
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
else
|
|
{
|
|
Assert.True(result.Failed);
|
|
Assert.Contains(expectedError, result.FailureMessage);
|
|
}
|
|
}
|
|
}
|