diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs index d00e1274..c5b7e832 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs @@ -255,11 +255,15 @@ try .GetValue(nameof(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.AllowOutsideDevelopment))); builder.Services.AddSecurity(disableLogin); builder.Services.AddCentralUI(); - // Local SQLite store by default; a local store replicating against a shared SQL-Server hub - // only when Secrets:Replication:Enabled is true AND a hub connection string is present. - // See SecretsRegistration for why the gate exists (eager options validation) and why the - // hub connection string can never itself come from the hub. - builder.Services.AddScadaBridgeSecrets(builder.Configuration); + // Local SQLite store by default. With Secrets:Replication:Enabled true it becomes either a + // local store synced against a shared SQL-Server hub (Mode=SqlServer, and only when a hub + // connection string is present) or — the production topology, scadaproj#3 — the HOST half + // of the pull-only gRPC secrets hub (Mode=Grpc), which central serves and sites sweep. + // The Central role never registers the sweep client; passing the role explicitly is what + // makes that a compile-time property of this composition root rather than a config guess. + // See SecretsRegistration for why the SQL gate exists (eager options validation), why the + // hub credential can never itself come from the hub, and why gRPC mode fails closed. + builder.Services.AddScadaBridgeSecrets(builder.Configuration, SecretsNodeRole.Central); // Secrets UI authorization: adds the named policies secrets:manage + secrets:reveal // (role-based) consumed by the /admin/secrets page. AddSecretsAuthorization only ADDS // these two policies via Configure — it composes additively with @@ -555,6 +559,22 @@ try // connection to the right pipeline by listener/protocol. app.MapGrpcService(); + // The pull-only secrets hub (scadaproj#3), on the SAME h2c listener and the same addressing + // convention — a site dials central's CentralGrpcPort, never central's database. It shares + // the listener with CentralControlService the way the site's LocalDb sync endpoint shares + // its one: two disjoint service prefixes, two independent fail-closed gates. The + // CentralControlAuthInterceptor registered on AddGrpc is prefix-scoped to + // CentralControlService and passes hub calls straight through; the hub's own + // SecretsHubAuthInterceptor — attached per-service by AddZbSecretsGrpcHub — is what gates + // it, on the shared Secrets:GrpcHub:BearerToken. + // + // A NO-OP unless this node registered the hub. That is not an optimisation: mapping the + // service without registering it would map a hub whose interceptor was never attached, + // i.e. an unauthenticated endpoint serving every secret central holds. Both this call and + // the registration above route through SecretsRegistration.UsesGrpcHub so they cannot + // drift apart. + app.MapScadaBridgeSecretsHub(app.Configuration); + app.MapStaticAssets(); app.MapCentralUI(); app.MapInboundAPI(); diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/SecretsRegistration.cs b/src/ZB.MOM.WW.ScadaBridge.Host/SecretsRegistration.cs index 9b9f75d6..a3ee18c7 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/SecretsRegistration.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/SecretsRegistration.cs @@ -1,8 +1,44 @@ +using Microsoft.AspNetCore.Routing; using ZB.MOM.WW.Secrets.DependencyInjection; +using ZB.MOM.WW.Secrets.Replicator.Grpc.DependencyInjection; using ZB.MOM.WW.Secrets.Replicator.SqlServer.DependencyInjection; namespace ZB.MOM.WW.ScadaBridge.Host; +/// +/// Which half of a clustered secrets topology this node composes. Passed in by the caller rather +/// than read from configuration, because both composition roots already know statically which one +/// they are — Program.cs for central and for a site — +/// and a role read from a key could be got wrong in a way that silently turns a site into a hub. +/// +public enum SecretsNodeRole +{ + /// The central cluster. Hosts the gRPC secrets hub; never sweeps from one. + Central, + + /// A site cluster. Sweeps from central's hub; never hosts one. + Site, +} + +/// +/// The clustered-replication transport, selected by Secrets:Replication:Mode. +/// +public enum SecretsReplicationMode +{ + /// + /// Local store bidirectionally synced with a shared SQL-Server hub. The historical mode and + /// therefore the default, so an existing configuration that sets only + /// Secrets:Replication:Enabled keeps behaving exactly as it did. + /// + SqlServer, + + /// + /// Local store on every node, converged by a pull-only gRPC hub that central hosts. The + /// production topology (scadaproj#3). + /// + Grpc, +} + /// /// Single composition-root entry point for the host container's secret store, shared by the /// central-role registrations in Program.cs and the site-role registrations in @@ -11,25 +47,45 @@ namespace ZB.MOM.WW.ScadaBridge.Host; /// /// /// Clustered replication is opt-in and off by default. ScadaBridge is hub-and-spoke — one -/// central cluster plus N separate site clusters — and hub mode gives each node a LOCAL store that -/// syncs bidirectionally with a shared central SQL hub, so a site keeps resolving secrets straight -/// through a WAN outage. +/// central cluster plus N separate site clusters — and every supported topology gives each node a +/// LOCAL store, so a site keeps resolving secrets straight through a WAN outage. What differs +/// between the two modes is only how that local store is kept converged. /// /// -/// The gate is not a style preference. AddZbSecretsSqlServerReplication validates its options -/// EAGERLY at registration time, so calling it unconditionally would throw at startup on every node -/// where Secrets:SqlServer:ConnectionString is unset — which is every dev box, every docker -/// node and every existing deployment. Both the explicit Secrets:Replication:Enabled flag and -/// a non-blank connection string are therefore required before the SQL-Server package is touched at -/// all; otherwise this is byte-for-byte the plain local-SQLite registration it has always been. +/// is the production topology (scadaproj#3): +/// central hosts a pull-only gRPC hub and each site node sweeps it. It exists because +/// requires every site node to hold a connection +/// string to central's database, which violates ScadaBridge's standing rule that sites talk to +/// central, not to central's DB. Pull-only is a property of the wire contract rather than a +/// setting: the proto carries no write RPC, so a site cannot push even if misconfigured, and +/// secrets therefore originate at central only. /// /// -/// Bootstrap constraint. The hub connection string is itself a secret, and it can never come -/// from the hub — a node cannot read the hub to learn how to reach the hub. It must arrive from -/// outside the replicated set: an environment variable, or a ${secret:} reference that is -/// seeded in that node's own LOCAL store (the pre-host expander in Program.cs runs against a -/// plain local SQLite store before the host container exists, so such a reference does resolve). -/// That expander is deliberately left un-replicated for exactly this reason. +/// The SQL-Server gate is not a style preference. AddZbSecretsSqlServerReplication +/// validates its options EAGERLY at registration time, so calling it unconditionally would throw at +/// startup on every node where Secrets:SqlServer:ConnectionString is unset — which is every +/// dev box, every docker node and every existing deployment. Both the explicit +/// Secrets:Replication:Enabled flag and a non-blank connection string are therefore required +/// before the SQL-Server package is touched at all; otherwise this is byte-for-byte the plain +/// local-SQLite registration it has always been. +/// +/// +/// The gRPC mode deliberately does NOT degrade the same way. A missing connection string in +/// SQL-Server mode falls back to a local-only store with a loud warning, and that behaviour is +/// preserved untouched. gRPC mode instead fails at startup when its endpoint or bearer token is +/// missing, because the fallback is the exact outcome the hub exists to prevent — a site serving +/// secrets that silently never converge with central — and because selecting the mode takes an +/// explicit second key, so an operator who typed it meant it. +/// +/// +/// Bootstrap constraint. A replication credential can never itself come from the replicated +/// set — a node cannot read the hub to learn how to reach the hub. The SQL hub connection string +/// must arrive from outside it: an environment variable, or a ${secret:} reference seeded in +/// that node's own LOCAL store (the pre-host expander in Program.cs runs against a plain +/// local SQLite store before the host container exists, so such a reference does resolve). The +/// same holds for Secrets:GrpcHub:BearerToken, which is supplied from appsettings or the +/// environment exactly as the mesh pre-shared keys are. That expander is deliberately left +/// un-replicated for exactly this reason. /// /// public static class SecretsRegistration @@ -40,28 +96,123 @@ public static class SecretsRegistration /// Configuration section holding the SQL-Server hub options. public const string SqlServerSectionPath = "Secrets:SqlServer"; + /// + /// Configuration section holding the gRPC hub options. One section serves both halves: central + /// reads BearerToken + MaxNamesPerRequest from it, a site reads Endpoint, + /// BearerToken and the sweep timings. + /// + public const string GrpcHubSectionPath = "Secrets:GrpcHub"; + /// Configuration key gating clustered replication. Absent or false = off. public const string ReplicationEnabledKey = "Secrets:Replication:Enabled"; + /// + /// Configuration key selecting the replication transport. Absent or blank = + /// , the historical behaviour. + /// + public const string ReplicationModeKey = "Secrets:Replication:Mode"; + /// Configuration key holding the shared SQL-Server hub connection string. public const string HubConnectionStringKey = "Secrets:SqlServer:ConnectionString"; + /// + /// Reads . An absent or blank value is + /// ; anything unrecognised throws. + /// + /// + /// The value is validated whenever it is present, even with replication switched off. A typo + /// only surfaces at the boot that introduces it that way — deferring the check to whenever + /// somebody later flips would fail a node in an unrelated + /// change, pointing at a key nobody touched. + /// + /// Application configuration. + /// The selected mode. + /// The configured value is not a known mode. + public static SecretsReplicationMode ResolveReplicationMode(IConfiguration config) + { + ArgumentNullException.ThrowIfNull(config); + + var raw = config[ReplicationModeKey]; + if (string.IsNullOrWhiteSpace(raw)) + { + return SecretsReplicationMode.SqlServer; + } + + if (!Enum.TryParse(raw.Trim(), ignoreCase: true, out SecretsReplicationMode mode) + || !Enum.IsDefined(mode)) + { + throw new InvalidOperationException( + $"{ReplicationModeKey} is '{raw}', which is not a known secrets replication mode. " + + $"Valid values are '{nameof(SecretsReplicationMode.SqlServer)}' and " + + $"'{nameof(SecretsReplicationMode.Grpc)}'."); + } + + return mode; + } + + /// + /// Whether this node participates in the pull-only gRPC secrets hub. + /// + /// + /// The single predicate behind BOTH the service registration and + /// , so the two can never disagree. That matters more + /// than tidiness: mapping the hub endpoint without registering it would map a service whose + /// fail-closed bearer interceptor was never attached — an unauthenticated hub serving every + /// secret central holds. + /// + /// Application configuration. + /// when replication is enabled in gRPC mode. + public static bool UsesGrpcHub(IConfiguration config) + { + ArgumentNullException.ThrowIfNull(config); + + return config.GetValue(ReplicationEnabledKey) + && ResolveReplicationMode(config) == SecretsReplicationMode.Grpc; + } + /// /// Registers the host container's secret store: a plain local SQLite store by default, or a - /// local store replicating against a shared SQL-Server hub when replication is explicitly - /// enabled and a hub connection string is present. + /// replicating topology when is set — a shared SQL-Server + /// hub, or the pull-only gRPC hub whose half is chosen by . /// /// The service collection to register into. /// Application configuration for options binding. + /// Which half of a clustered topology this node composes. /// The same instance, for chaining. public static IServiceCollection AddScadaBridgeSecrets( this IServiceCollection services, - IConfiguration config) + IConfiguration config, + SecretsNodeRole role) { ArgumentNullException.ThrowIfNull(services); ArgumentNullException.ThrowIfNull(config); var enabled = config.GetValue(ReplicationEnabledKey); + var mode = ResolveReplicationMode(config); + + if (enabled && mode == SecretsReplicationMode.Grpc) + { + // Neither gRPC extension registers the local store — pull-only means local writes + // never leave the node, so there is nothing to decorate and the store stays exactly as + // configured. Register it first; both halves resolve it. + services.AddZbSecrets(config, SecretsSectionPath); + + if (role == SecretsNodeRole.Central) + { + // Throws when Secrets:GrpcHub:BearerToken is unset. Deliberately not caught: a hub + // that starts and refuses every follower looks like a network fault from the site + // end, and would be discovered as a stale secret rather than as a boot failure. + services.AddZbSecretsGrpcHub(config, GrpcHubSectionPath); + } + else + { + // Throws when Secrets:GrpcHub:Endpoint or :BearerToken is unset. Same reasoning. + services.AddZbSecretsGrpcHubClient(config, GrpcHubSectionPath); + } + + return services; + } + var connectionString = config[HubConnectionStringKey]; if (enabled && !string.IsNullOrWhiteSpace(connectionString)) @@ -89,4 +240,34 @@ public static class SecretsRegistration return services.AddZbSecrets(config, SecretsSectionPath); } + + /// + /// Maps the central-hosted secrets hub endpoint, but only on a node whose configuration + /// actually registered it. A no-op otherwise. + /// + /// + /// Central-only by construction: registers the hub service + /// and its bearer interceptor solely for , and a site node + /// never reaches this call. The endpoint lands on the shared endpoint routing, so it is served + /// on the dedicated h2c control-plane listener (ScadaBridge:Node:CentralGrpcPort, + /// default 8083) — the same listener and the same addressing convention sites already use to + /// reach CentralControlService. + /// + /// The endpoint route builder. + /// Application configuration. + /// The same instance, for chaining. + public static IEndpointRouteBuilder MapScadaBridgeSecretsHub( + this IEndpointRouteBuilder endpoints, + IConfiguration config) + { + ArgumentNullException.ThrowIfNull(endpoints); + ArgumentNullException.ThrowIfNull(config); + + if (!UsesGrpcHub(config)) + { + return endpoints; + } + + return endpoints.MapZbSecretsHub(); + } } diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs b/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs index c4ff637e..87466574 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs @@ -105,11 +105,16 @@ public static class SiteServiceRegistration services.AddZbLocalDbReplication(config); services.AddDataConnectionLayer(); - // Local SQLite store by default; a local store replicating against a shared SQL-Server hub - // only when Secrets:Replication:Enabled is true AND a hub connection string is present. - // Hub mode matters most here: a site node keeps resolving secrets from its local store - // straight through a WAN outage to central. See SecretsRegistration. - services.AddScadaBridgeSecrets(config); + // Local SQLite store by default. With Secrets:Replication:Enabled true it becomes either a + // local store synced against a shared SQL-Server hub (Mode=SqlServer) or — the production + // topology, scadaproj#3 — the FOLLOWER half of the pull-only gRPC secrets hub (Mode=Grpc), + // sweeping central over the same gRPC path the control plane already uses instead of + // holding a connection string to central's database. A site never hosts the hub; passing + // the role explicitly is what makes that a property of this composition root. + // + // Either way the node keeps a FULL local store, so it resolves secrets straight through a + // WAN outage to central and simply converges late. See SecretsRegistration. + services.AddScadaBridgeSecrets(config, SecretsNodeRole.Site); // Adapter that surfaces the site id to // StoreAndForwardService through DI WITHOUT introducing a // StoreAndForward → HealthMonitoring project-reference cycle. Must be diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/appsettings.json b/src/ZB.MOM.WW.ScadaBridge.Host/appsettings.json index ccde2cb1..15f76930 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/appsettings.json +++ b/src/ZB.MOM.WW.ScadaBridge.Host/appsettings.json @@ -14,13 +14,28 @@ "MasterKey": { "Source": "Environment", "EnvVarName": "ZB_SECRETS_MASTER_KEY" }, "RunMigrationsOnStartup": true, "ResolveCacheTtl": "00:00:30", - "Replication": { "Enabled": false }, + "Replication": { + "_comment": "Default-OFF pin: no deployment enables clustered secret replication by shipping this file. Mode selects the transport when Enabled is true - 'SqlServer' (the default, kept so an existing config that sets only Enabled behaves exactly as before) or 'Grpc' (the PRODUCTION topology, scadaproj#3: central hosts a pull-only hub under Secrets:GrpcHub and sites sweep it, so a site never needs a connection string to central's database). Every node in either topology must carry the SAME ZB_SECRETS_MASTER_KEY - only ciphertext crosses the wire, so a node with a different KEK fails closed on resolve with a kek_id mismatch that reads like corruption but is a deployment error.", + "Enabled": false, + "Mode": "SqlServer" + }, "SqlServer": { "ConnectionString": "", "SchemaName": "zbsecrets", "SyncInterval": "00:00:30", "SyncOnStartup": true, - "_comment": "Hub connstr must be seeded in the LOCAL store (or supplied via env) - it cannot come from the hub itself. Enable via Secrets:Replication:Enabled." + "_comment": "Hub connstr must be seeded in the LOCAL store (or supplied via env) - it cannot come from the hub itself. Enable via Secrets:Replication:Enabled + Mode=SqlServer." + }, + "GrpcHub": { + "_comment": "Read ONLY when Secrets:Replication:Enabled is true AND Mode=Grpc; inert otherwise. ONE section, both halves: CENTRAL reads BearerToken + MaxNamesPerRequest and hosts the hub on its CentralGrpcPort h2c listener (default 8083, alongside CentralControlService); a SITE reads Endpoint + BearerToken + the sweep timings and pulls. Replication is pull-only by wire contract - the proto has no write RPC - so secrets originate at central and a site cannot push. FAIL-CLOSED: unlike SqlServer mode there is no local-only fallback; a missing BearerToken (either role) or Endpoint (site) is a startup failure naming the key.", + "_bearerToken": "Shared credential every follower presents. Supply it from appsettings or the environment (Secrets__GrpcHub__BearerToken), NOT as a ${secret:...} reference - resolving that reference is what the hub exists to make possible, so it cannot come from the hub. Same rationale and same handling as the mesh pre-shared keys. Never commit a real value here; the empty default below is fail-closed, not open.", + "BearerToken": "", + "_endpoint": "SITE ONLY. Absolute http/https URI of a CENTRAL node's gRPC (h2c) port - the same address family as ScadaBridge:Communication:CentralGrpcEndpoints, e.g. 'http://central-a-host:8083'. NOT via Traefik (HTTP/1 only). NOTE the asymmetry with CentralGrpcEndpoints: that is a LIST and fails over across the central pair, whereas the hub client dials a SINGLE endpoint. A sweep against a downed central-a therefore stalls rather than failing over - which is survivable because the sweep is best-effort (one warning per interval, then retry) and the site keeps serving its full local last-known-good store, but it does mean secrets stop converging until that central node returns.", + "Endpoint": "", + "SyncInterval": "00:00:30", + "SyncOnStartup": true, + "CallDeadline": "00:00:30", + "MaxNamesPerRequest": 1000 } }, "Serilog": {