Files
ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Host/StartupValidator.cs
T
Joseph Doherty 2e4e41a8f7 fix(auditlog): site audit DB onto the data volume; required path + soft flush
Closes WP1.2 of the arch-review remediation plan (finding #2, High):
SqliteAuditWriterOptions.DatabasePath defaulted to CWD-relative "auditlog.db",
which on the docker rig resolves onto the container's ephemeral overlayfs
(not the mounted /app/data volume), silently discarding the pending audit
forward-state backlog on every recreate; nothing in docker/ or docker-env2/
overrode it; FlushIntervalMs was validated but never read by the writer loop
(one commit per event even at trickle rate); and no PRAGMA synchronous was
set (SQLite's FULL default fsyncs every commit).

- DatabasePath now has no default (mirrors ZB.MOM.WW.LocalDb's LocalDbOptions.Path)
  and is required pre-host for Site nodes only, via a new StartupValidator raw-config
  check (top-level "AuditLog:SiteWriter:DatabasePath", NOT nested under ScadaBridge:
  AddAuditLog binds that section off the configuration root). SqliteAuditWriterOptionsValidator
  deliberately does NOT check DatabasePath itself, because AddAuditLog runs its
  ValidateOnStart on both Central and Site composition roots but only Site nodes
  ever resolve the writer — checking it there would fail Central's boot too.
- All 8 site-node appsettings under docker/ and docker-env2/ now set
  AuditLog:SiteWriter:DatabasePath to /app/data/auditlog.db (mounted volume,
  survives container recreate, same convention as LocalDb:Path); the local-dev
  base appsettings.Site.json sets ./data/auditlog.db to match.
- The writer loop now honors FlushIntervalMs: after draining the immediately
  available burst, it keeps the transaction open (bounded by FlushIntervalMs
  from the first event) waiting for more trickle-rate events before committing,
  instead of flushing (and fsyncing) per event.
- PRAGMA synchronous = NORMAL on the write connection — audit is best-effort by
  design (CLAUDE.md: "Audit-write failure NEVER aborts the user-facing action"),
  so NORMAL's narrower power-loss window is an acceptable trade for far fewer
  fsyncs; WAL mode still guarantees no corruption.
- Tests: StartupValidator site-required/blank/central-exempt cases; writer
  trickle-load single-transaction coalescing + beyond-interval separate-transaction
  regression (new FlushCountForTests seam); options-validator doc updates reflecting
  the moved responsibility. Full suite runs green: AuditLog.Tests 368/368,
  Host.Tests 480/480.

One-time migration note: the existing container-local auditlog.db (wherever it
landed under CWD) is abandoned by this change, not migrated — already-forwarded
rows are safe centrally (AuditLog is the durable copy), and any still-Pending
rows on the abandoned path are lost once. This is the exact bug being fixed, not
a new loss: those rows were already living outside the mounted volume and would
not have survived the next container recreate regardless. Cross-reference
docs/known-issues/2026-07-20-cached-telemetry-drain-hot-loop.md, which this
placement bug caused.
2026-08-14 20:13:31 -04:00

259 lines
16 KiB
C#

using ZB.MOM.WW.Configuration;
namespace ZB.MOM.WW.ScadaBridge.Host;
/// <summary>
/// Validates required configuration before Akka.NET actor system creation.
/// Runs early in startup to fail fast with clear error messages.
/// </summary>
public static class StartupValidator
{
/// <summary>Validates required configuration values and throws <see cref="InvalidOperationException"/> listing all errors if any are found.</summary>
/// <param name="configuration">The application configuration to validate.</param>
public static void Validate(IConfiguration configuration)
{
// Resolve the same locals the original imperative validator used, so the
// cross-field predicates below can close over them. ConfigPreflight.Require
// passes config[key] to each predicate, but the cross-field rules ignore that
// argument and read these resolved values instead — preserving the exact
// conditions (and therefore the byte-identical failure messages and ordering)
// of the original StartupValidator.
var nodeSection = configuration.GetSection("ScadaBridge:Node");
var role = nodeSection["Role"];
var portStr = nodeSection["RemotingPort"];
bool portValid = int.TryParse(portStr, out var port) && port >= 1 && port <= 65535;
var seedNodes = configuration.GetSection("ScadaBridge:Cluster:SeedNodes").Get<List<string>>();
// GrpcPort: default 8083 when absent; only fails the range rule when the key is
// present AND invalid. The out-param assignment mirrors the original so the
// resolved grpcPort feeds the cross-field rules even on a parse failure.
var grpcPortStr = nodeSection["GrpcPort"];
int grpcPort = 8083; // NodeOptions default when the key is absent
bool grpcValid = !(grpcPortStr != null && (!int.TryParse(grpcPortStr, out grpcPort) || grpcPort < 1 || grpcPort > 65535));
// MetricsPort: default 8084 when absent; same parse-or-default contract as GrpcPort.
var metricsPortStr = nodeSection["MetricsPort"];
int metricsPort = 8084; // NodeOptions default when the key is absent
bool metricsValid = !(metricsPortStr != null && (!int.TryParse(metricsPortStr, out metricsPort) || metricsPort < 1 || metricsPort > 65535));
ConfigPreflight.For(configuration)
// Role / NodeHostname / RemotingPort (unconditional)
.Require("ScadaBridge:Node:Role",
_ => !(string.IsNullOrEmpty(role) || (role != "Central" && role != "Site")),
"must be 'Central' or 'Site'")
.Require("ScadaBridge:Node:NodeHostname",
_ => !string.IsNullOrEmpty(nodeSection["NodeHostname"]),
"is required")
.Require("ScadaBridge:Node:RemotingPort",
_ => portValid,
"must be 1-65535")
// SiteId (Site only) — note: OUTSIDE the big Site block in the original,
// so it must run before the unconditional SeedNodes-count rule.
.When(role == "Site", p => p
.Require("ScadaBridge:Node:SiteId",
_ => !string.IsNullOrEmpty(nodeSection["SiteId"]),
"is required for Site nodes"))
// Central-only database/security rules.
.When(role == "Central", p => p
.Require("ScadaBridge:Database:ConfigurationDb",
_ => !string.IsNullOrEmpty(configuration.GetSection("ScadaBridge:Database")["ConfigurationDb"]),
"connection string required for Central")
.Require("ScadaBridge:Database:MachineDataDb",
_ => !string.IsNullOrEmpty(configuration.GetSection("ScadaBridge:Database")["MachineDataDb"]),
"connection string required for Central")
// The LDAP server key moved into the nested Security:Ldap
// sub-section (bound to the shared LdapOptions). Validate the nested key so
// the pre-host preflight still fails fast on a missing LDAP server for
// Central. The full LDAP option set (SearchBase / ServiceAccountDn /
// transport) is additionally validated post-host by the shared
// LdapOptionsValidator (registered with ValidateOnStart by AddZbLdapAuth).
.Require("ScadaBridge:Security:Ldap:Server",
_ => !string.IsNullOrEmpty(configuration.GetSection("ScadaBridge:Security:Ldap")["Server"]),
"required for Central")
.Require("ScadaBridge:Security:JwtSigningKey",
_ => !string.IsNullOrEmpty(configuration.GetSection("ScadaBridge:Security")["JwtSigningKey"]),
"required for Central")
// The inbound API-key pepper
// backs the peppered-HMAC secret compare in the shared
// ZB.MOM.WW.Auth.ApiKeys verifier (wired by AddZbApiKeyAuth at the
// Central composition root). A missing or too-short pepper does not
// fault at boot — the verifier just fails every secret compare, so the
// inbound API silently serves 401s to otherwise-valid keys. Validate it
// here (Central-only, pre-host) so a misconfigured pepper fails fast at
// startup with a clear message instead of as a runtime auth blackout.
// The Require predicate receives config[key] directly; the >=16-char
// floor matches the test pepper's minimum and the secret-strength
// baseline used elsewhere.
.Require("ScadaBridge:InboundApi:ApiKeyPepper",
value => !string.IsNullOrEmpty(value) && value.Length >= 16,
"is required and must be at least 16 characters for Central (backs the inbound API-key peppered-HMAC verifier)"))
// SeedNodes count (unconditional, after SiteId).
.Require("ScadaBridge:Cluster:SeedNodes",
_ => seedNodes != null && seedNodes.Count >= 2,
"must have at least 2 entries")
// Self-first seed ordering (decision 2026-07-22). Akka runs FirstSeedNodeProcess —
// the ONLY bootstrap path that can form a new cluster when no peer answers InitJoin —
// exclusively when seed-nodes[0] is this node's own address. Every other node runs
// JoinSeedNodeProcess and retries InitJoin forever, so a node listing its partner
// first cannot cold-start alone: that is the "registered outage gap"
// (docker/README.md), and it is a silent failure at boot rather than a loud one.
// Enforced here rather than in ClusterOptionsValidator because only this validator
// sees both the node identity and the seed list.
.Require("ScadaBridge:Cluster:SeedNodes",
_ => seedNodes is not { Count: >= 1 }
|| SeedNodeIsSelf(seedNodes[0], nodeSection["NodeHostname"], port),
"must list this node itself first: seed-nodes[0] has to be this node's own "
+ $"'akka.tcp://scadabridge@{nodeSection["NodeHostname"]}:{port}'. Akka only lets "
+ "seed-nodes[0] form a new cluster, so with the partner listed first this node "
+ "can never start while its peer is down (Component-ClusterInfrastructure.md → "
+ "Seed Node Ordering)")
// The big Site-only block: GrpcPort/MetricsPort validity + cross-field
// collisions + seed-node-port loop, in the original order.
.When(role == "Site", p =>
{
// GrpcPort range, then GrpcPort vs RemotingPort.
p.Require("ScadaBridge:Node:GrpcPort", _ => grpcValid, "must be 1-65535");
// Identical GrpcPort/RemotingPort make Kestrel and Akka.Remote contend
// for the same TCP port. Uses the resolved GrpcPort, including 8083.
p.Require("ScadaBridge:Node:GrpcPort", _ => port != grpcPort, "must differ from RemotingPort");
// MetricsPort range, then MetricsPort vs both ports.
p.Require("ScadaBridge:Node:MetricsPort", _ => metricsValid, "must be 1-65535");
// The Kestrel metrics (HTTP/1.1) listener port must differ from BOTH the
// Akka remoting port and the gRPC port. Uses the resolved MetricsPort (8084 default).
p.Require("ScadaBridge:Node:MetricsPort", _ => metricsPort != port, "must differ from RemotingPort");
p.Require("ScadaBridge:Node:MetricsPort", _ => metricsPort != grpcPort, "must differ from GrpcPort");
// The gRPC control-plane preshared key. Same argument as the inbound
// API-key pepper above: without it the node boots and looks healthy, but
// ControlPlaneAuthInterceptor is fail-closed, so every SiteStream call —
// live subscriptions, audit pulls, cached-telemetry ingest — is refused
// with PermissionDenied. A silent, total loss of the site's central-facing
// surface is far worse than a loud boot failure, so require it here.
// Central holds the matching value per site in its secret store as
// SB-GRPC-PSK-{SiteId}; production supplies this one as
// ${secret:SB-GRPC-PSK-<siteId>}, expanded before the host is built.
p.Require("ScadaBridge:Communication:GrpcPsk",
value => !string.IsNullOrWhiteSpace(value),
"is required for Site nodes: it is the preshared key the gRPC control "
+ "plane authenticates with, and the interceptor is fail-closed, so an "
+ "unset key refuses every SiteStream call. Set the same value here (in "
+ "production as ${secret:SB-GRPC-PSK-<siteId>}) and under the secret "
+ "name SB-GRPC-PSK-<siteId> in central's secret store");
// gRPC (CentralControlService) is the only site→central transport after the
// ClusterClient→gRPC migration's Phase 4 — the Akka ClusterClient path and its
// CentralContactPoints option are gone. A site with no central gRPC endpoint has
// nothing to dial: heartbeats, health reports, notification forwards and audit
// ingest all silently fail. The shared CommunicationOptionsValidator only rejects
// BLANK entries (it is role-agnostic, and central nodes legitimately leave the list
// empty), so the "a Site must have at least one" rule lives here, where the role is
// known. The predicate reads index :0 directly, so its non-empty presence proves the
// list has a usable first endpoint.
p.Require("ScadaBridge:Communication:CentralGrpcEndpoints:0",
value => !string.IsNullOrWhiteSpace(value),
"is required for Site nodes: gRPC (CentralControlService) is the only "
+ "site→central transport, so each site must list at least one central gRPC "
+ "endpoint under ScadaBridge:Communication:CentralGrpcEndpoints "
+ "(e.g. http://scadabridge-central-a:8083). Central nodes leave it empty.");
// The site hot-path audit writer's SQLite file (arch-review remediation
// WP1.2). SqliteAuditWriterOptions.DatabasePath has no default (mirrors
// LocalDb:Path) — an unset value would previously fall back to a bare
// "auditlog.db" resolved relative to the process CWD, which on the docker
// rig is the container's ephemeral overlayfs, not the mounted /app/data
// volume: the file, and every pending (not-yet-forwarded) audit row in it,
// was silently discarded on every container recreate. AddAuditLog binds
// this options type on BOTH roles (SqliteAuditWriterOptionsValidator
// deliberately does not check DatabasePath there, so Central's boot is
// unaffected — see that validator's remarks), so the Site-only requirement
// lives here, the same way GrpcPsk is gated just above. NOTE: unlike every
// other key in this method, AuditLog:SiteWriter is a TOP-LEVEL config
// section (AddAuditLog binds "AuditLog:SiteWriter" off the configuration
// root, not "ScadaBridge:AuditLog:SiteWriter") — no ScadaBridge: prefix.
p.Require("AuditLog:SiteWriter:DatabasePath",
value => !string.IsNullOrWhiteSpace(value),
"is required for Site nodes: the SQLite hot-path audit writer has no "
+ "default path (mirrors LocalDb:Path) — an unset value would silently "
+ "resolve to a CWD-relative file on the container's ephemeral overlayfs "
+ "and lose the pending audit backlog on every redeploy. Point it at the "
+ "mounted data volume, e.g. /app/data/auditlog.db.");
// ScadaBridge:Database:SiteDbPath was required here until LocalDb
// Phase 2. The site's tables now live in the consolidated LocalDb
// database (LocalDb:Path, which SiteServiceRegistration requires),
// and SiteDbPath survives only as the legacy migration source — so
// its absence means "nothing to migrate", not a misconfiguration.
// DatabaseOptionsValidator still rejects a present-but-blank value.
// A seed node must reference an Akka.Remote endpoint, never the
// Kestrel HTTP/2 gRPC port. A seed entry whose port equals this node's
// GrpcPort would make a joining node attempt an Akka.Remote TCP
// association against the gRPC listener and fail.
foreach (var seed in seedNodes ?? Enumerable.Empty<string>())
{
p.Require("ScadaBridge:Cluster:SeedNodes",
_ => SeedNodePort(seed) != grpcPort,
$"entry '{seed}' must not target the gRPC port " +
$"({grpcPort}); seed nodes must reference Akka remoting ports");
// Same failure mode as the gRPC guard: the Kestrel HTTP/1.1
// metrics listener is not an Akka.Remote endpoint, so a seed on
// it dials a doomed association forever.
p.Require("ScadaBridge:Cluster:SeedNodes",
_ => SeedNodePort(seed) != metricsPort,
$"entry '{seed}' must not target the metrics port " +
$"({metricsPort}); seed nodes must reference Akka remoting ports");
}
})
.ThrowIfInvalid();
}
/// <summary>
/// Extracts the TCP port from an Akka seed-node address of the form
/// <c>akka.tcp://system@host:port</c>. Returns <c>-1</c> when no port can be parsed.
/// </summary>
private static int SeedNodePort(string seedNode)
{
if (string.IsNullOrWhiteSpace(seedNode))
return -1;
var lastColon = seedNode.LastIndexOf(':');
if (lastColon < 0 || lastColon == seedNode.Length - 1)
return -1;
return int.TryParse(seedNode[(lastColon + 1)..], out var port) ? port : -1;
}
/// <summary>
/// Extracts the host from an Akka seed-node address of the form
/// <c>akka.tcp://system@host:port</c>. Returns an empty string when no host can be parsed.
/// </summary>
private static string SeedNodeHost(string seedNode)
{
if (string.IsNullOrWhiteSpace(seedNode))
return string.Empty;
var at = seedNode.LastIndexOf('@');
var lastColon = seedNode.LastIndexOf(':');
if (at < 0 || lastColon <= at)
return string.Empty;
return seedNode[(at + 1)..lastColon];
}
/// <summary>
/// True when <paramref name="seedNode"/> addresses this node itself (host AND port).
/// Host comparison is case-insensitive because DNS names are; it is otherwise exact —
/// Akka does no DNS canonicalisation either, so <c>node-a</c> and
/// <c>node-a.example.com</c> are genuinely different seed identities to the cluster.
/// </summary>
private static bool SeedNodeIsSelf(string seedNode, string? nodeHostname, int remotingPort)
{
return SeedNodePort(seedNode) == remotingPort
&& string.Equals(SeedNodeHost(seedNode), nodeHostname, StringComparison.OrdinalIgnoreCase);
}
}