Merge branch 'worktree-agent-af8a44154c4e2dbba' into arch-review-remediation
This commit is contained in:
@@ -78,5 +78,16 @@
|
|||||||
// Replication is opt-in and configured separately; absent = local-only.
|
// Replication is opt-in and configured separately; absent = local-only.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "/app/data/site-localdb.db"
|
"Path": "/app/data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,5 +78,16 @@
|
|||||||
// Replication is opt-in and configured separately; absent = local-only.
|
// Replication is opt-in and configured separately; absent = local-only.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "/app/data/site-localdb.db"
|
"Path": "/app/data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -120,5 +120,16 @@
|
|||||||
"MaxOplogRows": 250000,
|
"MaxOplogRows": 250000,
|
||||||
"MaxOplogAge": "2.00:00:00"
|
"MaxOplogAge": "2.00:00:00"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -113,5 +113,16 @@
|
|||||||
"MaxOplogRows": 250000,
|
"MaxOplogRows": 250000,
|
||||||
"MaxOplogAge": "2.00:00:00"
|
"MaxOplogAge": "2.00:00:00"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,5 +86,16 @@
|
|||||||
// Replication is opt-in and configured separately; absent = local-only.
|
// Replication is opt-in and configured separately; absent = local-only.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "/app/data/site-localdb.db"
|
"Path": "/app/data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,5 +86,16 @@
|
|||||||
// Replication is opt-in and configured separately; absent = local-only.
|
// Replication is opt-in and configured separately; absent = local-only.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "/app/data/site-localdb.db"
|
"Path": "/app/data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,5 +86,16 @@
|
|||||||
// Replication is opt-in and configured separately; absent = local-only.
|
// Replication is opt-in and configured separately; absent = local-only.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "/app/data/site-localdb.db"
|
"Path": "/app/data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,5 +86,16 @@
|
|||||||
// Replication is opt-in and configured separately; absent = local-only.
|
// Replication is opt-in and configured separately; absent = local-only.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "/app/data/site-localdb.db"
|
"Path": "/app/data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) and StartupValidator now requires it for
|
||||||
|
// Site nodes — an unset value used to fall back to a bare "auditlog.db" resolved
|
||||||
|
// relative to CWD, i.e. the container's ephemeral overlayfs, so the pending audit
|
||||||
|
// backlog was silently discarded on every recreate. On the mounted /app/data volume,
|
||||||
|
// same as LocalDb:Path, so it survives container recreate.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "/app/data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -174,6 +174,24 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
|||||||
pragmaCmd.ExecuteNonQuery();
|
pragmaCmd.ExecuteNonQuery();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// synchronous=NORMAL (arch-review remediation WP1.2): SQLite's FULL default
|
||||||
|
// fsyncs on every transaction commit, which is the dominant cost of the
|
||||||
|
// per-event write path this same change coalesces via FlushIntervalMs — FULL
|
||||||
|
// would still fsync once per FLUSH even after that fix. NORMAL fsyncs at
|
||||||
|
// fewer, WAL-checkpoint-aligned points instead of every commit; the window
|
||||||
|
// this opens is a handful of the most recent commits lost on OS crash / power
|
||||||
|
// loss (WAL mode itself still guarantees no *corruption*, only a possible
|
||||||
|
// short rollback on unclean restart). That trade is acceptable here because
|
||||||
|
// audit writes are explicitly best-effort by design (CLAUDE.md: "Audit-write
|
||||||
|
// failure NEVER aborts the user-facing action — audit is best-effort, the
|
||||||
|
// action's own success/failure path is authoritative") — this is the same
|
||||||
|
// durability class as a dropped audit row, not a new risk class.
|
||||||
|
using (var pragmaCmd = _connection.CreateCommand())
|
||||||
|
{
|
||||||
|
pragmaCmd.CommandText = "PRAGMA synchronous = NORMAL";
|
||||||
|
pragmaCmd.ExecuteNonQuery();
|
||||||
|
}
|
||||||
|
|
||||||
// Enable FK enforcement on the WRITE connection. PRAGMA foreign_keys is
|
// Enable FK enforcement on the WRITE connection. PRAGMA foreign_keys is
|
||||||
// a per-connection, per-session setting in SQLite — it is NOT persisted
|
// a per-connection, per-session setting in SQLite — it is NOT persisted
|
||||||
// in the database file, so every new connection that may INSERT into
|
// in the database file, so every new connection that may INSERT into
|
||||||
@@ -306,20 +324,67 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
|||||||
{
|
{
|
||||||
batch.Clear();
|
batch.Clear();
|
||||||
batch.Add(first);
|
batch.Add(first);
|
||||||
|
var deadline = DateTime.UtcNow.AddMilliseconds(_options.FlushIntervalMs);
|
||||||
|
|
||||||
// Pull additional ready events up to BatchSize. TryRead is non-
|
// Pull additional ready events up to BatchSize. TryRead is non-
|
||||||
// blocking and lets us amortise the transaction overhead across a
|
// blocking and lets us amortise the transaction overhead across a
|
||||||
// burst of concurrent enqueues.
|
// burst of concurrent enqueues. Once the immediately-available
|
||||||
while (batch.Count < _options.BatchSize &&
|
// burst is drained, keep the transaction open for up to
|
||||||
_writeQueue.Reader.TryRead(out var next))
|
// FlushIntervalMs waiting for MORE events to trickle in — this is
|
||||||
|
// the soft-flush coalescing arch-review remediation WP1.2 adds:
|
||||||
|
// without it, a low-rate trickle of writes (one script call every
|
||||||
|
// few ms, well under BatchSize) fsyncs once PER EVENT, because the
|
||||||
|
// writer loop reaches an empty channel and flushes immediately.
|
||||||
|
// FlushIntervalMs bounds the worst-case added latency any single
|
||||||
|
// event can see from this coalescing.
|
||||||
|
while (batch.Count < _options.BatchSize)
|
||||||
{
|
{
|
||||||
batch.Add(next);
|
if (_writeQueue.Reader.TryRead(out var next))
|
||||||
|
{
|
||||||
|
batch.Add(next);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var remaining = deadline - DateTime.UtcNow;
|
||||||
|
if (remaining <= TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
var waitToRead = _writeQueue.Reader.WaitToReadAsync().AsTask();
|
||||||
|
var completed = await Task.WhenAny(
|
||||||
|
waitToRead, Task.Delay(remaining)).ConfigureAwait(false);
|
||||||
|
|
||||||
|
if (completed != waitToRead)
|
||||||
|
{
|
||||||
|
// Deadline elapsed with nothing new — flush what we have.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitToReadAsync resolving false means the channel completed
|
||||||
|
// (Dispose) with nothing left to read; resolving true means at
|
||||||
|
// least one item is available, so the top of the loop's
|
||||||
|
// TryRead will pick it up.
|
||||||
|
if (!await waitToRead.ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
FlushBatch(batch);
|
FlushBatch(batch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Total number of <see cref="FlushBatch"/> transactions committed since
|
||||||
|
/// construction. Test-only observability seam (arch-review remediation
|
||||||
|
/// WP1.2) for asserting that FlushIntervalMs coalesces a trickle of
|
||||||
|
/// near-simultaneous writes into a single transaction rather than one
|
||||||
|
/// commit per event; production code never reads this.
|
||||||
|
/// </summary>
|
||||||
|
internal int FlushCountForTests => _flushCountForTests;
|
||||||
|
private int _flushCountForTests;
|
||||||
|
|
||||||
private void FlushBatch(IReadOnlyList<PendingAuditEvent> batch)
|
private void FlushBatch(IReadOnlyList<PendingAuditEvent> batch)
|
||||||
{
|
{
|
||||||
lock (_writeLock)
|
lock (_writeLock)
|
||||||
@@ -436,6 +501,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
transaction.Commit();
|
transaction.Commit();
|
||||||
|
_flushCountForTests++;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -9,8 +9,23 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class SqliteAuditWriterOptions
|
public sealed class SqliteAuditWriterOptions
|
||||||
{
|
{
|
||||||
/// <summary>SQLite database path (or in-memory URI for tests).</summary>
|
/// <summary>
|
||||||
public string DatabasePath { get; set; } = "auditlog.db";
|
/// SQLite database path (or in-memory URI for tests). No default —
|
||||||
|
/// mirrors <c>LocalDb:Path</c> (<c>ZB.MOM.WW.LocalDb.LocalDbOptions.Path</c>): a
|
||||||
|
/// bare filename here would resolve relative to the process CWD, which on the
|
||||||
|
/// docker rig is the container's ephemeral overlayfs, not the mounted
|
||||||
|
/// <c>/app/data</c> volume — the file (and its pending forward-state backlog)
|
||||||
|
/// would be silently discarded on every container recreate. Site nodes MUST set
|
||||||
|
/// this explicitly (e.g. <c>/app/data/auditlog.db</c>); <c>StartupValidator</c>
|
||||||
|
/// in ZB.MOM.WW.ScadaBridge.Host enforces the requirement pre-host for Site nodes only,
|
||||||
|
/// the same way it enforces <c>Communication:GrpcPsk</c>. Central composition roots also
|
||||||
|
/// bind this options type (<c>AddAuditLog</c> is shared between roles) but never resolve
|
||||||
|
/// <see cref="SqliteAuditWriter"/>, so the empty default is inert there and deliberately
|
||||||
|
/// left unvalidated by <see cref="SqliteAuditWriterOptionsValidator"/> (which only checks
|
||||||
|
/// the role-agnostic channel/batch/flush knobs) — enforcing it there too would fail
|
||||||
|
/// central's boot via <c>ValidateOnStart</c>, since <c>AddAuditLog</c> runs on both roles.
|
||||||
|
/// </summary>
|
||||||
|
public string DatabasePath { get; set; } = "";
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Capacity of the bounded write queue. Set high enough that ordinary
|
/// Capacity of the bounded write queue. Set high enough that ordinary
|
||||||
|
|||||||
@@ -6,19 +6,25 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
|||||||
/// Validates <see cref="SqliteAuditWriterOptions"/> at host startup
|
/// Validates <see cref="SqliteAuditWriterOptions"/> at host startup
|
||||||
/// (arch-review 08 round 2 NF4). The channel/batch/flush knobs feed the
|
/// (arch-review 08 round 2 NF4). The channel/batch/flush knobs feed the
|
||||||
/// background writer task; a zero anywhere stalls it (nothing drains, nothing
|
/// background writer task; a zero anywhere stalls it (nothing drains, nothing
|
||||||
/// flushes), and an empty <c>DatabasePath</c> leaves the SQLite writer with no
|
/// flushes). <see cref="SqliteAuditWriterOptions.BacklogPollIntervalSeconds"/>
|
||||||
/// backing store. <see cref="SqliteAuditWriterOptions.BacklogPollIntervalSeconds"/>
|
|
||||||
/// is intentionally NOT validated — a non-positive value has a documented
|
/// is intentionally NOT validated — a non-positive value has a documented
|
||||||
/// fall-back-to-30s contract in <c>SiteAuditBacklogReporter</c>.
|
/// fall-back-to-30s contract in <c>SiteAuditBacklogReporter</c>.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <see cref="SqliteAuditWriterOptions.DatabasePath"/> is deliberately NOT
|
||||||
|
/// validated here (arch-review remediation WP1.2). <c>AddAuditLog</c> runs on
|
||||||
|
/// BOTH Central and Site composition roots and binds this options type with
|
||||||
|
/// <c>ValidateOnStart</c> either way, but only Site nodes ever resolve
|
||||||
|
/// <see cref="SqliteAuditWriter"/> — requiring a non-empty path in this
|
||||||
|
/// role-agnostic validator would fail Central's boot too. The requirement is
|
||||||
|
/// instead enforced pre-host, Site-role-only, by <c>StartupValidator</c> in
|
||||||
|
/// ZB.MOM.WW.ScadaBridge.Host (mirrors how it gates <c>Communication:GrpcPsk</c>).
|
||||||
|
/// </remarks>
|
||||||
public sealed class SqliteAuditWriterOptionsValidator : OptionsValidatorBase<SqliteAuditWriterOptions>
|
public sealed class SqliteAuditWriterOptionsValidator : OptionsValidatorBase<SqliteAuditWriterOptions>
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
protected override void Validate(ValidationBuilder builder, SqliteAuditWriterOptions options)
|
protected override void Validate(ValidationBuilder builder, SqliteAuditWriterOptions options)
|
||||||
{
|
{
|
||||||
builder.RequireThat(!string.IsNullOrWhiteSpace(options.DatabasePath),
|
|
||||||
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.DatabasePath)} must be a non-empty path.");
|
|
||||||
|
|
||||||
builder.RequireThat(options.ChannelCapacity > 0,
|
builder.RequireThat(options.ChannelCapacity > 0,
|
||||||
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.ChannelCapacity)} " +
|
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.ChannelCapacity)} " +
|
||||||
$"({options.ChannelCapacity}) must be > 0.");
|
$"({options.ChannelCapacity}) must be > 0.");
|
||||||
|
|||||||
@@ -159,6 +159,28 @@ public static class StartupValidator
|
|||||||
+ "endpoint under ScadaBridge:Communication:CentralGrpcEndpoints "
|
+ "endpoint under ScadaBridge:Communication:CentralGrpcEndpoints "
|
||||||
+ "(e.g. http://scadabridge-central-a:8083). Central nodes leave it empty.");
|
+ "(e.g. http://scadabridge-central-a:8083). Central nodes leave it empty.");
|
||||||
|
|
||||||
|
// The site hot-path audit writer's SQLite file (arch-review remediation
|
||||||
|
// WP1.2). SqliteAuditWriterOptions.DatabasePath has no default (mirrors
|
||||||
|
// LocalDb:Path) — an unset value would previously fall back to a bare
|
||||||
|
// "auditlog.db" resolved relative to the process CWD, which on the docker
|
||||||
|
// rig is the container's ephemeral overlayfs, not the mounted /app/data
|
||||||
|
// volume: the file, and every pending (not-yet-forwarded) audit row in it,
|
||||||
|
// was silently discarded on every container recreate. AddAuditLog binds
|
||||||
|
// this options type on BOTH roles (SqliteAuditWriterOptionsValidator
|
||||||
|
// deliberately does not check DatabasePath there, so Central's boot is
|
||||||
|
// unaffected — see that validator's remarks), so the Site-only requirement
|
||||||
|
// lives here, the same way GrpcPsk is gated just above. NOTE: unlike every
|
||||||
|
// other key in this method, AuditLog:SiteWriter is a TOP-LEVEL config
|
||||||
|
// section (AddAuditLog binds "AuditLog:SiteWriter" off the configuration
|
||||||
|
// root, not "ScadaBridge:AuditLog:SiteWriter") — no ScadaBridge: prefix.
|
||||||
|
p.Require("AuditLog:SiteWriter:DatabasePath",
|
||||||
|
value => !string.IsNullOrWhiteSpace(value),
|
||||||
|
"is required for Site nodes: the SQLite hot-path audit writer has no "
|
||||||
|
+ "default path (mirrors LocalDb:Path) — an unset value would silently "
|
||||||
|
+ "resolve to a CWD-relative file on the container's ephemeral overlayfs "
|
||||||
|
+ "and lose the pending audit backlog on every redeploy. Point it at the "
|
||||||
|
+ "mounted data volume, e.g. /app/data/auditlog.db.");
|
||||||
|
|
||||||
// ScadaBridge:Database:SiteDbPath was required here until LocalDb
|
// ScadaBridge:Database:SiteDbPath was required here until LocalDb
|
||||||
// Phase 2. The site's tables now live in the consolidated LocalDb
|
// Phase 2. The site's tables now live in the consolidated LocalDb
|
||||||
// database (LocalDb:Path, which SiteServiceRegistration requires),
|
// database (LocalDb:Path, which SiteServiceRegistration requires),
|
||||||
|
|||||||
@@ -80,5 +80,14 @@
|
|||||||
// fails to boot, so every site config must set it.
|
// fails to boot, so every site config must set it.
|
||||||
"LocalDb": {
|
"LocalDb": {
|
||||||
"Path": "./data/site-localdb.db"
|
"Path": "./data/site-localdb.db"
|
||||||
|
},
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path above) - AuditLog:SiteWriter:DatabasePath is
|
||||||
|
// REQUIRED and validated pre-host on Site nodes (StartupValidator), the same way
|
||||||
|
// LocalDb:Path is, so every site config must set it.
|
||||||
|
"AuditLog": {
|
||||||
|
"SiteWriter": {
|
||||||
|
"DatabasePath": "./data/auditlog.db"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-12
@@ -5,30 +5,30 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Eager startup validation for <see cref="SqliteAuditWriterOptions"/>
|
/// Eager startup validation for <see cref="SqliteAuditWriterOptions"/>
|
||||||
/// (arch-review 08 round 2 NF4). The site hot-path writer's channel/batch/flush
|
/// (arch-review 08 round 2 NF4). The site hot-path writer's channel/batch/flush
|
||||||
/// knobs must be positive or the background writer task cannot make progress;
|
/// knobs must be positive or the background writer task cannot make progress.
|
||||||
/// an empty <c>DatabasePath</c> would leave the SQLite writer with nowhere to go.
|
|
||||||
/// <c>BacklogPollIntervalSeconds</c> is deliberately NOT validated — a
|
/// <c>BacklogPollIntervalSeconds</c> is deliberately NOT validated — a
|
||||||
/// non-positive value has a documented fall-back-to-30s contract.
|
/// non-positive value has a documented fall-back-to-30s contract.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <c>DatabasePath</c> is intentionally NOT covered here (arch-review remediation
|
||||||
|
/// WP1.2): it defaults to <c>""</c> (no CWD-relative default — mirrors
|
||||||
|
/// <c>LocalDb:Path</c>), but this validator runs on both Central and Site
|
||||||
|
/// composition roots via <c>ValidateOnStart</c>, and only Site nodes ever resolve
|
||||||
|
/// the writer. The Site-only requirement is enforced pre-host by
|
||||||
|
/// <c>StartupValidator</c> in ZB.MOM.WW.ScadaBridge.Host — see
|
||||||
|
/// <c>StartupValidatorTests.SiteWithoutAuditLogDatabasePath_FailsValidation</c>.
|
||||||
|
/// </remarks>
|
||||||
public class SqliteAuditWriterOptionsValidatorTests
|
public class SqliteAuditWriterOptionsValidatorTests
|
||||||
{
|
{
|
||||||
[Fact]
|
[Fact]
|
||||||
public void DefaultOptions_AreValid()
|
public void DefaultOptions_AreValid()
|
||||||
{
|
{
|
||||||
|
// Includes the empty DatabasePath default — this validator does not check
|
||||||
|
// it (see class remarks); only StartupValidator does, and only for Site nodes.
|
||||||
var validator = new SqliteAuditWriterOptionsValidator();
|
var validator = new SqliteAuditWriterOptionsValidator();
|
||||||
Assert.True(validator.Validate(null, new SqliteAuditWriterOptions()).Succeeded);
|
Assert.True(validator.Validate(null, new SqliteAuditWriterOptions()).Succeeded);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void EmptyDatabasePath_Fails()
|
|
||||||
{
|
|
||||||
var validator = new SqliteAuditWriterOptionsValidator();
|
|
||||||
var result = validator.Validate(null, new SqliteAuditWriterOptions { DatabasePath = "" });
|
|
||||||
Assert.False(result.Succeeded);
|
|
||||||
Assert.Contains(result.Failures!,
|
|
||||||
f => f.Contains(nameof(SqliteAuditWriterOptions.DatabasePath), StringComparison.Ordinal));
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ZeroChannelCapacity_Fails()
|
public void ZeroChannelCapacity_Fails()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ public class SqliteAuditWriterWriteTests
|
|||||||
private static (SqliteAuditWriter writer, string dataSource) CreateWriter(
|
private static (SqliteAuditWriter writer, string dataSource) CreateWriter(
|
||||||
string testName,
|
string testName,
|
||||||
int? channelCapacity = null,
|
int? channelCapacity = null,
|
||||||
INodeIdentityProvider? nodeIdentity = null)
|
INodeIdentityProvider? nodeIdentity = null,
|
||||||
|
int? flushIntervalMs = null)
|
||||||
{
|
{
|
||||||
var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared";
|
var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared";
|
||||||
var opts = new SqliteAuditWriterOptions { DatabasePath = dataSource };
|
var opts = new SqliteAuditWriterOptions { DatabasePath = dataSource };
|
||||||
@@ -32,6 +33,10 @@ public class SqliteAuditWriterWriteTests
|
|||||||
{
|
{
|
||||||
opts.ChannelCapacity = cap;
|
opts.ChannelCapacity = cap;
|
||||||
}
|
}
|
||||||
|
if (flushIntervalMs is int flush)
|
||||||
|
{
|
||||||
|
opts.FlushIntervalMs = flush;
|
||||||
|
}
|
||||||
|
|
||||||
// Default identity provider returns null — existing tests pre-date
|
// Default identity provider returns null — existing tests pre-date
|
||||||
// SourceNode stamping and have no expectation about it. New stamping
|
// SourceNode stamping and have no expectation about it. New stamping
|
||||||
@@ -181,6 +186,69 @@ public class SqliteAuditWriterWriteTests
|
|||||||
Assert.Equal(1000, Convert.ToInt64(sidecarCmd.ExecuteScalar()));
|
Assert.Equal(1000, Convert.ToInt64(sidecarCmd.ExecuteScalar()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----- FlushIntervalMs soft-flush coalescing (arch-review remediation WP1.2) ----- //
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A trickle of events — well under <see cref="SqliteAuditWriterOptions.BatchSize"/>,
|
||||||
|
/// spaced closer together than <see cref="SqliteAuditWriterOptions.FlushIntervalMs"/> —
|
||||||
|
/// must coalesce into ONE transaction rather than fsyncing once per event. This is the
|
||||||
|
/// bug WP1.2 fixes: <c>FlushIntervalMs</c> was validated but never read by the writer
|
||||||
|
/// loop, so every trickle-rate write incurred its own commit (and, under SQLite's
|
||||||
|
/// default synchronous=FULL, its own fsync).
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task WriteAsync_TrickleLoad_CoalescesIntoOneTransaction()
|
||||||
|
{
|
||||||
|
var (writer, dataSource) = CreateWriter(
|
||||||
|
nameof(WriteAsync_TrickleLoad_CoalescesIntoOneTransaction),
|
||||||
|
flushIntervalMs: 500);
|
||||||
|
await using var _ = writer;
|
||||||
|
|
||||||
|
const int count = 5;
|
||||||
|
var writeTasks = new List<Task>(count);
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
writeTasks.Add(writer.WriteAsync(NewEvent()));
|
||||||
|
await Task.Delay(15); // spaced well under the 500ms flush window
|
||||||
|
}
|
||||||
|
await Task.WhenAll(writeTasks);
|
||||||
|
|
||||||
|
using var connection = OpenVerifierConnection(dataSource);
|
||||||
|
using var cmd = connection.CreateCommand();
|
||||||
|
cmd.CommandText = "SELECT COUNT(*) FROM audit_event;";
|
||||||
|
Assert.Equal(count, Convert.ToInt64(cmd.ExecuteScalar()));
|
||||||
|
|
||||||
|
// The defining assertion: all 5 trickled-in writes landed in a single
|
||||||
|
// FlushBatch transaction, not up to 5 separate commits.
|
||||||
|
Assert.Equal(1, writer.FlushCountForTests);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The flip side: two events spaced FURTHER apart than
|
||||||
|
/// <see cref="SqliteAuditWriterOptions.FlushIntervalMs"/> must NOT be held open
|
||||||
|
/// waiting for a partner — the deadline bounds worst-case added latency, it does
|
||||||
|
/// not turn into an unbounded debounce.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task WriteAsync_EventsSpacedBeyondFlushInterval_FlushSeparately()
|
||||||
|
{
|
||||||
|
var (writer, dataSource) = CreateWriter(
|
||||||
|
nameof(WriteAsync_EventsSpacedBeyondFlushInterval_FlushSeparately),
|
||||||
|
flushIntervalMs: 50);
|
||||||
|
await using var _ = writer;
|
||||||
|
|
||||||
|
await writer.WriteAsync(NewEvent());
|
||||||
|
await Task.Delay(300); // well beyond the 50ms flush window
|
||||||
|
await writer.WriteAsync(NewEvent());
|
||||||
|
|
||||||
|
using var connection = OpenVerifierConnection(dataSource);
|
||||||
|
using var cmd = connection.CreateCommand();
|
||||||
|
cmd.CommandText = "SELECT COUNT(*) FROM audit_event;";
|
||||||
|
Assert.Equal(2, Convert.ToInt64(cmd.ExecuteScalar()));
|
||||||
|
|
||||||
|
Assert.Equal(2, writer.FlushCountForTests);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task WriteAsync_DuplicateEventId_FirstWriteWins_NoException()
|
public async Task WriteAsync_DuplicateEventId_FirstWriteWins_NoException()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -27,10 +27,12 @@ public class SiteHealthEndpointTests : IDisposable
|
|||||||
private readonly List<IDisposable> _disposables = new();
|
private readonly List<IDisposable> _disposables = new();
|
||||||
private readonly Dictionary<string, string?> _previousEnv = new(StringComparer.Ordinal);
|
private readonly Dictionary<string, string?> _previousEnv = new(StringComparer.Ordinal);
|
||||||
private readonly string _tempDbPath;
|
private readonly string _tempDbPath;
|
||||||
|
private readonly string _tempAuditDbPath;
|
||||||
|
|
||||||
public SiteHealthEndpointTests()
|
public SiteHealthEndpointTests()
|
||||||
{
|
{
|
||||||
_tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_ep_{Guid.NewGuid()}.db");
|
_tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_ep_{Guid.NewGuid()}.db");
|
||||||
|
_tempAuditDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_ep_audit_{Guid.NewGuid()}.db");
|
||||||
|
|
||||||
// Whole-key env overrides, the sanctioned path: supplying GrpcPsk concretely makes the
|
// Whole-key env overrides, the sanctioned path: supplying GrpcPsk concretely makes the
|
||||||
// pre-host secrets expander skip appsettings.Site.json's ${secret:SB-GRPC-PSK-site-1}
|
// pre-host secrets expander skip appsettings.Site.json's ${secret:SB-GRPC-PSK-site-1}
|
||||||
@@ -50,6 +52,11 @@ public class SiteHealthEndpointTests : IDisposable
|
|||||||
SetEnv("ScadaBridge__Cluster__SeedNodes__1", "akka.tcp://scadabridge@localhost:18085");
|
SetEnv("ScadaBridge__Cluster__SeedNodes__1", "akka.tcp://scadabridge@localhost:18085");
|
||||||
SetEnv("ScadaBridge__Communication__GrpcPsk", "test-psk-0123456789");
|
SetEnv("ScadaBridge__Communication__GrpcPsk", "test-psk-0123456789");
|
||||||
SetEnv("LocalDb__Path", _tempDbPath);
|
SetEnv("LocalDb__Path", _tempDbPath);
|
||||||
|
// arch-review remediation WP1.2: SqliteAuditWriterOptions.DatabasePath has no default
|
||||||
|
// and StartupValidator now requires it for Site nodes — same reason LocalDb__Path is
|
||||||
|
// overridden above (appsettings.Site.json's own default is CWD-relative and would
|
||||||
|
// otherwise litter the test working directory when SiteAuditBacklogReporter probes it).
|
||||||
|
SetEnv("AuditLog__SiteWriter__DatabasePath", _tempAuditDbPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void SetEnv(string key, string? value)
|
private void SetEnv(string key, string? value)
|
||||||
@@ -71,6 +78,7 @@ public class SiteHealthEndpointTests : IDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
try { File.Delete(_tempDbPath); } catch { /* best effort */ }
|
try { File.Delete(_tempDbPath); } catch { /* best effort */ }
|
||||||
|
try { File.Delete(_tempAuditDbPath); } catch { /* best effort */ }
|
||||||
|
|
||||||
GC.SuppressFinalize(this);
|
GC.SuppressFinalize(this);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,6 +44,10 @@ public class StartupValidatorTests
|
|||||||
// Phase 4: gRPC (CentralControlService) is the only site→central transport, so a Site
|
// Phase 4: gRPC (CentralControlService) is the only site→central transport, so a Site
|
||||||
// node must list at least one central gRPC endpoint to dial (no Akka fallback remains).
|
// node must list at least one central gRPC endpoint to dial (no Akka fallback remains).
|
||||||
["ScadaBridge:Communication:CentralGrpcEndpoints:0"] = "http://central-a:8083",
|
["ScadaBridge:Communication:CentralGrpcEndpoints:0"] = "http://central-a:8083",
|
||||||
|
// arch-review remediation WP1.2: the site hot-path audit writer's SQLite file has no
|
||||||
|
// default path (mirrors LocalDb:Path) — required so it lands on the mounted data volume,
|
||||||
|
// not the container's ephemeral overlayfs.
|
||||||
|
["AuditLog:SiteWriter:DatabasePath"] = "/app/data/auditlog.db",
|
||||||
};
|
};
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -124,6 +128,43 @@ public class StartupValidatorTests
|
|||||||
Assert.Null(Record.Exception(() => StartupValidator.Validate(config)));
|
Assert.Null(Record.Exception(() => StartupValidator.Validate(config)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SiteWithoutAuditLogDatabasePath_FailsValidation()
|
||||||
|
{
|
||||||
|
// arch-review remediation WP1.2: SqliteAuditWriterOptions.DatabasePath has no default
|
||||||
|
// (mirrors LocalDb:Path). An unset value used to fall back to a bare "auditlog.db"
|
||||||
|
// resolved relative to CWD — on the docker rig, the container's ephemeral overlayfs —
|
||||||
|
// so the node booted, looked healthy, and lost its pending audit backlog on every
|
||||||
|
// recreate. Fail fast at boot instead.
|
||||||
|
var values = ValidSiteConfig();
|
||||||
|
values.Remove("AuditLog:SiteWriter:DatabasePath");
|
||||||
|
var config = BuildConfig(values);
|
||||||
|
|
||||||
|
var ex = Assert.Throws<InvalidOperationException>(() => StartupValidator.Validate(config));
|
||||||
|
Assert.Contains("DatabasePath", ex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SiteWithBlankAuditLogDatabasePath_FailsValidation()
|
||||||
|
{
|
||||||
|
var values = ValidSiteConfig();
|
||||||
|
values["AuditLog:SiteWriter:DatabasePath"] = " ";
|
||||||
|
var config = BuildConfig(values);
|
||||||
|
|
||||||
|
var ex = Assert.Throws<InvalidOperationException>(() => StartupValidator.Validate(config));
|
||||||
|
Assert.Contains("DatabasePath", ex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CentralWithoutAuditLogDatabasePath_PassesValidation()
|
||||||
|
{
|
||||||
|
// AddAuditLog binds SqliteAuditWriterOptions on both roles, but only Site nodes ever
|
||||||
|
// resolve the writer — the requirement must not fire for Central.
|
||||||
|
var config = BuildConfig(ValidCentralConfig());
|
||||||
|
|
||||||
|
Assert.Null(Record.Exception(() => StartupValidator.Validate(config)));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void MissingRole_FailsValidation()
|
public void MissingRole_FailsValidation()
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user