feat(secrets): Secrets:Store:Provider=SqlServer — central pair shares one SQL secret store without the hub

This commit is contained in:
Joseph Doherty
2026-09-28 17:44:14 -04:00
parent ebc55c5f72
commit 35235ecfed
5 changed files with 202 additions and 10 deletions
@@ -131,6 +131,17 @@ public static class SecretsRegistration
/// <summary>Configuration key holding the shared SQL-Server hub connection string.</summary>
public const string HubConnectionStringKey = "Secrets:SqlServer:ConnectionString";
/// <summary>
/// Configuration key selecting a CENTRAL node's own local store provider — independent of
/// <see cref="ReplicationEnabledKey"/>/<see cref="ReplicationModeKey"/> and the gRPC hub.
/// Absent or blank = <c>Sqlite</c> (per-node, the historical default); <c>SqlServer</c> points
/// both central nodes at ONE shared SQL-Server database directly, with no hub in between —
/// for a fleet that ships secrets to sites on every artifact deploy, a clustered central needs
/// both nodes to see the same secret set, and this is the lighter-weight way to get that
/// without standing up the gRPC hub. A Site node always ignores this key.
/// </summary>
public const string StoreProviderKey = "Secrets:Store:Provider";
/// <summary>
/// Reads <see cref="ReplicationModeKey"/>. An absent or blank value is
/// <see cref="SecretsReplicationMode.SqlServer"/>; anything unrecognised throws.
@@ -319,6 +330,28 @@ public static class SecretsRegistration
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(config);
// Secrets:Store:Provider=SqlServer on Central, BEFORE the replication branch below and
// gated OFF whenever clustered replication is enabled (that is a different, existing
// feature — the gRPC hub or SQL-Server replication already decide central's store in that
// case). A Site never reaches this: it always keeps its local SQLite store, straight
// through a WAN outage, same as every other topology.
if (role == SecretsNodeRole.Central
&& string.Equals(config[StoreProviderKey], "SqlServer", StringComparison.OrdinalIgnoreCase)
&& !config.GetValue<bool>(ReplicationEnabledKey))
{
// Same fail-closed rules as the Grpc-mode central branch below: blank, or a
// ${secret:} reference (bootstrap circularity — the pre-host expander needs this
// same store to resolve it).
EnsureCentralSharedStoreConnectionString(config);
// Registers the shared SQL-Server store as ISecretStore directly — no hub, no
// follower sweep. Both central nodes just point at the same database, so they serve
// identical manifests by construction, the same guarantee the Grpc-mode branch gets
// from the hub, without standing one up.
return services.AddZbSecretsSqlServerStore(
config, SecretsSectionPath, SqlServerSectionPath);
}
// UsesGrpcHub is the single predicate shared with MapScadaBridgeSecretsHub — the branch
// condition must never be re-derived here, or registration and endpoint mapping could
// disagree about whether the hub is on.
@@ -97,6 +97,17 @@ public static class StartupValidator
value => string.IsNullOrWhiteSpace(value)
|| (Enum.TryParse<InboundApiKeyStoreProvider>(value, ignoreCase: true, out var provider)
&& Enum.IsDefined(provider)),
"must be 'Sqlite' or 'SqlServer' when set")
// The central shared secret store provider (Secrets:Store:Provider — a fleet that
// ships secrets to sites on every artifact deploy needs a clustered central pair to
// see the same secret set). Not required — an unset value defaults to Sqlite,
// per-node, mirroring the inbound API-key store default above — but when it IS set,
// it must be one of the two legal values so a typo fails fast at boot instead of
// silently keeping the per-node SQLite store it looks like it opted out of.
.Require("Secrets:Store:Provider",
value => string.IsNullOrWhiteSpace(value)
|| string.Equals(value, "Sqlite", StringComparison.OrdinalIgnoreCase)
|| string.Equals(value, "SqlServer", StringComparison.OrdinalIgnoreCase),
"must be 'Sqlite' or 'SqlServer' when set"))
// SeedNodes count (unconditional, after SiteId).
.Require("ScadaBridge:Cluster:SeedNodes",
@@ -226,20 +237,33 @@ public static class StartupValidator
var role = configuration.GetSection("ScadaBridge:Node")["Role"];
var seedNodes = configuration.GetSection("ScadaBridge:Cluster:SeedNodes").Get<List<string>>();
var provider = InboundApiKeyStoreOptions.ReadProvider(configuration);
var secretsStoreProvider = configuration["Secrets:Store:Provider"];
var isClusteredCentral = role == "Central" && seedNodes is { Count: >= 2 };
if (role == "Central"
&& seedNodes is { Count: >= 2 }
&& provider == InboundApiKeyStoreProvider.Sqlite)
var warnings = new List<string>();
if (isClusteredCentral && provider == InboundApiKeyStoreProvider.Sqlite)
{
return new[]
{
$"Inbound API keys use the per-node SQLite store on a {seedNodes.Count}-node central "
warnings.Add(
$"Inbound API keys use the per-node SQLite store on a {seedNodes!.Count}-node central "
+ "cluster; a failover will reject every inbound client until keys are re-issued. Set "
+ "ScadaBridge:InboundApi:ApiKeyStore:Provider=SqlServer (design 2026-09-02)."
};
+ "ScadaBridge:InboundApi:ApiKeyStore:Provider=SqlServer (design 2026-09-02).");
}
return Array.Empty<string>();
// Secrets:Store:Provider defaults to Sqlite (per-node) when unset, mirroring the inbound
// API-key store default above — so this fires both when the key is absent and when it is
// explicitly "Sqlite". A fleet that ships secrets to sites on every artifact deploy needs
// both central nodes to deploy the SAME secret set; a per-node store means each node has
// silently drifted the moment anyone edits a secret on only one of them.
if (isClusteredCentral
&& !string.Equals(secretsStoreProvider, "SqlServer", StringComparison.OrdinalIgnoreCase))
{
warnings.Add(
"Secrets use a per-node SQLite store on a clustered central — each node would "
+ "deploy its own secret set to sites; set Secrets:Store:Provider=SqlServer");
}
return warnings;
}
/// <summary>
@@ -14,6 +14,10 @@
"MasterKey": { "Source": "Environment", "EnvVarName": "ZB_SECRETS_MASTER_KEY" },
"RunMigrationsOnStartup": true,
"ResolveCacheTtl": "00:00:30",
"Store": {
"_comment": "CENTRAL ONLY (a Site always ignores this and keeps its local SQLite store). Independent of Replication above and the gRPC hub - no hub is stood up either way. 'Sqlite' (default) is per-node: on a clustered central, each node's secrets can silently drift the moment anyone edits one on only a single node, which then ships an inconsistent secret set to sites on artifact deploy - StartupValidator warns on a >=2-seed central left on Sqlite. 'SqlServer' points BOTH central nodes at ONE shared SQL-Server database directly (Secrets:SqlServer:ConnectionString, same fail-closed blank/${secret:} rules as Grpc mode's shared store), so the pair can never diverge without standing up the gRPC hub.",
"Provider": "Sqlite"
},
"Replication": {
"_comment": "Default-OFF pin: no deployment enables clustered secret replication by shipping this file. Mode selects the transport when Enabled is true - 'SqlServer' (the default, kept so an existing config that sets only Enabled behaves exactly as before) or 'Grpc' (the PRODUCTION topology, scadaproj#3: central hosts a pull-only hub under Secrets:GrpcHub and sites sweep it, so a site never needs a connection string to central's database). Every node in either topology must carry the SAME ZB_SECRETS_MASTER_KEY - only ciphertext crosses the wire, so a node with a different KEK fails closed on resolve with a kek_id mismatch that reads like corruption but is a deployment error.",
"Enabled": false,