feat(secrets): bump ZB.MOM.WW.Secrets family to 0.5.0 and wire hub fallback endpoints

Pin all five ZB.MOM.WW.Secrets* packages 0.4.1 -> 0.5.0, which brings
SecretsGrpcHubClientOptions.FallbackEndpoints and the package's internal
FailoverSecretsHubReader. A site whose GrpcHub section lists fallback
endpoints now fails a sweep over to the next central instead of stalling
on a downed primary - safe ONLY because both central nodes serve one
shared SQL secret store (scadaproj#4), so either hub answers with the
same manifest; the appsettings comments say so and warn against listing
endpoints backed by independent stores.

appsettings.json gains "FallbackEndpoints": [] with a _fallbackEndpoints
comment, and the _endpoint note's single-endpoint-stall caveat is scoped
to the empty-list case it now only applies to.

Wiring pins (red first on 0.4.1): site + Grpc + one fallback resolves
ISecretsHubReader to FailoverSecretsHubReader with the
"zb-secrets-grpc-hub:fallback:0" keyed channel present; zero fallbacks
keeps the plain GrpcSecretsHubClient and no fallback channel - the
pre-0.5.0 container shape byte-identical.

Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1

Directory.Packages.props

@@ -104,11 +104,11 @@
     <PackageVersion Include="ZB.MOM.WW.Auth.AspNetCore" Version="0.1.5" />
     <PackageVersion Include="ZB.MOM.WW.Audit" Version="0.1.0" />
     <PackageVersion Include="ZB.MOM.WW.Theme" Version="0.3.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets" Version="0.4.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Abstractions" Version="0.4.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Ui" Version="0.4.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Replicator.SqlServer" Version="0.4.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Replicator.Grpc" Version="0.4.1" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets" Version="0.5.0" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Abstractions" Version="0.5.0" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Ui" Version="0.5.0" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Replicator.SqlServer" Version="0.5.0" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Replicator.Grpc" Version="0.5.0" />
     <PackageVersion Include="ZB.MOM.WW.LocalDb" Version="0.1.3" />
     <PackageVersion Include="ZB.MOM.WW.LocalDb.Replication" Version="0.1.3" />
     <PackageVersion Include="ZB.MOM.WW.LocalDb.Contracts" Version="0.1.3" />

src/ZB.MOM.WW.ScadaBridge.Host/Program.cs

@@ -32,6 +32,8 @@ using ZB.MOM.WW.ScadaBridge.Transport;
 using ZB.MOM.WW.Secrets.Abstractions;
 using ZB.MOM.WW.Secrets.Configuration;
 using ZB.MOM.WW.Secrets.DependencyInjection;
+using ZB.MOM.WW.Secrets.Replicator.SqlServer;
+using ZB.MOM.WW.Secrets.Replicator.SqlServer.DependencyInjection;
 using ZB.MOM.WW.Secrets.Sqlite;
 using ZB.MOM.WW.Secrets.Ui;
 using ZB.MOM.WW.Telemetry;
@@ -52,13 +54,56 @@ var configuration = new ConfigurationBuilder()
 
 // Expand ${secret:...} config references before any validator/binder sees them (Layer A).
 // Throwaway provider — disposed here, shares no singletons with the host container.
+//
+// The expander must read the SAME store the running node will serve. On a central node in Grpc
+// replication mode that store is the SHARED SQL-Server store (see SecretsRegistration,
+// scadaproj#4) — an expander left on SQLite there would resolve pre-host ${secret:} references
+// from a stale/empty local store, silently diverging from what the node's own hub serves. Every
+// other case (sites, SqlServer mode, replication off) keeps the local SQLite path exactly as it
+// always was. A blank or ${secret:}-valued connection string fails HERE, through the same
+// EnsureCentralSharedStoreConnectionString check AddScadaBridgeSecrets runs — this is the
+// earliest code that would otherwise hand the bad value to SqlConnection, whose generic
+// "initialization string" format error would bury the designed message.
+//
+// Two consequences worth knowing. This block runs BEFORE Serilog exists and outside the
+// try/catch below, so a SQL Server outage at central boot exits with a bare stderr stack
+// trace, no structured log — honest and container-restart-retryable, just unenriched; the
+// SQLite path had the same window but its only failure mode was a local file. And the
+// role check is deliberately NOT folded into SecretsRegistration: that class refuses to
+// read the node role from configuration (a config-read role could silently turn a site
+// into a hub — see SecretsNodeRole), while Program.cs must read it to branch the whole
+// composition anyway, so the config read stays here at the composition root.
+var expanderUsesSharedSqlStore =
+    string.Equals(
+        configuration["ScadaBridge:Node:Role"], "Central", StringComparison.OrdinalIgnoreCase)
+    && SecretsRegistration.UsesGrpcHub(configuration);
+
+var expanderServices = new ServiceCollection();
+if (expanderUsesSharedSqlStore)
+{
+    SecretsRegistration.EnsureCentralSharedStoreConnectionString(configuration);
+    expanderServices.AddZbSecretsSqlServerStore(configuration, "Secrets");
+}
+else
+{
+    expanderServices.AddZbSecrets(configuration, "Secrets");
+}
+
 #pragma warning disable ASP0000 // deliberate throwaway container
-await using (var secretsProvider = new ServiceCollection()
-    .AddZbSecrets(configuration, "Secrets")
-    .BuildServiceProvider())
+await using (var secretsProvider = expanderServices.BuildServiceProvider())
 #pragma warning restore ASP0000
 {
-    await secretsProvider.GetRequiredService<SqliteSecretsStoreMigrator>().MigrateAsync(default);
+    if (expanderUsesSharedSqlStore)
+    {
+        await secretsProvider.GetRequiredService<SqlServerSecretsStoreMigrator>()
+            .MigrateAsync(default);
+    }
+    else
+    {
+        await secretsProvider.GetRequiredService<SqliteSecretsStoreMigrator>()
+            .MigrateAsync(default);
+    }
+
     var resolver = secretsProvider.GetRequiredService<ISecretResolver>();
     await new SecretReferenceExpander(resolver)
         .ExpandConfigurationAsync(configuration, default);

src/ZB.MOM.WW.ScadaBridge.Host/SecretsRegistration.cs

@@ -47,9 +47,10 @@ public enum SecretsReplicationMode
 /// <remarks>
 /// <para>
 /// Clustered replication is <b>opt-in and off by default</b>. ScadaBridge is hub-and-spoke — one
-/// central cluster plus N separate site clusters — and every supported topology gives each node a
-/// LOCAL store, so a site keeps resolving secrets straight through a WAN outage. What differs
-/// between the two modes is only how that local store is kept converged.
+/// central cluster plus N separate site clusters — and every supported topology gives each SITE
+/// node a LOCAL store, so a site keeps resolving secrets straight through a WAN outage. What
+/// differs between the modes is how convergence works, and — in gRPC mode — where central's own
+/// store lives.
 /// </para>
 /// <para>
 /// <b><see cref="SecretsReplicationMode.Grpc"/> is the production topology</b> (scadaproj#3):
@@ -58,7 +59,17 @@ public enum SecretsReplicationMode
 /// string to central's database, which violates ScadaBridge's standing rule that sites talk to
 /// central, not to central's DB. Pull-only is a property of the wire contract rather than a
 /// setting: the proto carries no write RPC, so a site cannot push even if misconfigured, and
-/// secrets therefore originate at central only.
+/// secrets therefore originate at central only. <b>Central's own store in this mode is the SHARED
+/// SQL-Server store</b> (<c>AddZbSecretsSqlServerStore</c>), not local SQLite: both central nodes
+/// read and write ONE copy of every row, so the two hub instances serve identical manifests by
+/// construction. The alternative — an independent local store per central node — is the recorded
+/// defect (scadaproj#4): one central answering authenticated followers with an empty manifest,
+/// turning site-side hub failover into a silent convergence stop. The accepted trade is that
+/// central secret resolution now depends on SQL Server availability — acceptable because central
+/// is already SQL-coupled to its core, and a SQL outage makes the hub <em>throw</em> (sites keep
+/// last-known-good), never serve empty. The pre-host <c>${secret:}</c> expander in
+/// <c>Program.cs</c> follows this store swap, so pre-host references resolve from the same store
+/// the running node serves.
 /// </para>
 /// <para>
 /// <b>The SQL-Server gate is not a style preference.</b> <c>AddZbSecretsSqlServerReplication</c>
@@ -73,19 +84,23 @@ public enum SecretsReplicationMode
 /// <b>The gRPC mode deliberately does NOT degrade the same way.</b> A missing connection string in
 /// SQL-Server mode falls back to a local-only store with a loud warning, and that behaviour is
 /// preserved untouched. gRPC mode instead fails at startup when its endpoint or bearer token is
-/// missing, because the fallback is the exact outcome the hub exists to prevent — a site serving
-/// secrets that silently never converge with central — and because selecting the mode takes an
-/// explicit second key, so an operator who typed it meant it.
+/// missing — or, on central, when the shared-store connection string is — because the fallback is
+/// the exact outcome the hub exists to prevent — a site serving secrets that silently never
+/// converge with central — and because selecting the mode takes an explicit second key, so an
+/// operator who typed it meant it.
 /// </para>
 /// <para>
 /// <b>Bootstrap constraint.</b> A replication credential can never itself come from the replicated
-/// set — a node cannot read the hub to learn how to reach the hub. The SQL hub connection string
-/// must arrive from outside it: an environment variable, or a <c>${secret:}</c> reference seeded in
-/// that node's own LOCAL store (the pre-host expander in <c>Program.cs</c> runs against a plain
-/// local SQLite store before the host container exists, so such a reference does resolve). The
+/// set — a node cannot read the hub to learn how to reach the hub. In SQL-Server mode the hub
+/// connection string must arrive from outside it: an environment variable, or a <c>${secret:}</c>
+/// reference seeded in that node's own LOCAL store (the pre-host expander in <c>Program.cs</c>
+/// runs against the local SQLite store before the host container exists, so such a reference does
+/// resolve). In gRPC mode the rule is stricter on central: <c>Secrets:SqlServer:ConnectionString</c>
+/// can never be a <c>${secret:}</c> reference at all, because on central the pre-host expander
+/// itself runs against the shared SQL store (see <c>Program.cs</c>) and would need this very
+/// string to reach the store that resolves it — registration rejects such a value outright. The
 /// same holds for <c>Secrets:GrpcHub:BearerToken</c>, which is supplied from appsettings or the
-/// environment exactly as the mesh pre-shared keys are. That expander is deliberately left
-/// un-replicated for exactly this reason.
+/// environment exactly as the mesh pre-shared keys are.
 /// </para>
 /// </remarks>
 public static class SecretsRegistration
@@ -170,10 +185,67 @@ public static class SecretsRegistration
             && ResolveReplicationMode(config) == SecretsReplicationMode.Grpc;
     }
 
+    /// <summary>
+    /// The one definition of "a Central node in Grpc mode has a usable shared-store connection
+    /// string", shared by <see cref="AddScadaBridgeSecrets"/> and the Layer-A expander in
+    /// <c>Program.cs</c> so the two can never diverge. The expander runs FIRST on a real boot —
+    /// without this sharing, a bad value would die inside <c>SqlConnection</c> with a generic
+    /// "initialization string" format error before the designed message ever fired.
+    /// </summary>
+    /// <param name="config">Application configuration.</param>
+    /// <returns>The validated connection string.</returns>
+    /// <exception cref="InvalidOperationException">
+    /// <see cref="HubConnectionStringKey"/> is blank or contains a <c>${secret:}</c> reference.
+    /// </exception>
+    internal static string EnsureCentralSharedStoreConnectionString(IConfiguration config)
+    {
+        ArgumentNullException.ThrowIfNull(config);
+
+        var hubConnectionString = config[HubConnectionStringKey];
+
+        if (string.IsNullOrWhiteSpace(hubConnectionString))
+        {
+            // Fail closed, matching the mode's philosophy: central in Grpc mode REQUIRES the
+            // shared SQL store. An independent local store per central node is exactly the
+            // divergence scadaproj#4 recorded — one central hub answered authenticated followers
+            // with an EMPTY manifest, so a hub failover would "succeed" against nothing and stop
+            // convergence silently.
+            throw new InvalidOperationException(
+                $"{HubConnectionStringKey} is empty, but this node is Central with "
+                    + $"{ReplicationModeKey}=Grpc. Central in Grpc mode requires the "
+                    + "SHARED SQL-Server secret store — an independent local store per "
+                    + "central node is the divergence scadaproj#4 recorded (a hub "
+                    + "failover would 'succeed' against an empty manifest). Supply the "
+                    + "connection string via appsettings or the environment "
+                    + "(Secrets__SqlServer__ConnectionString).");
+        }
+
+        if (hubConnectionString.Contains("${secret:", StringComparison.Ordinal))
+        {
+            // The secrets store's own connection string can never be a secret reference: the
+            // pre-host expander needs the store this string points at in order to resolve it —
+            // bootstrap circularity, the same rule the class remarks document for the hub bearer
+            // token. The message deliberately does not echo the configured value: a mistyped
+            // value here could be a pasted real credential.
+            throw new InvalidOperationException(
+                $"{HubConnectionStringKey} contains a ${{secret:}} reference. The "
+                    + "secret store's own connection string can never be a secret "
+                    + "reference — the pre-host expander needs the store this string "
+                    + "points at in order to resolve it (bootstrap circularity, the "
+                    + "same rule as Secrets:GrpcHub:BearerToken). Supply a literal "
+                    + "value via appsettings or the environment "
+                    + "(Secrets__SqlServer__ConnectionString).");
+        }
+
+        return hubConnectionString;
+    }
+
     /// <summary>
     /// Registers the host container's secret store: a plain local SQLite store by default, or a
     /// replicating topology when <see cref="ReplicationEnabledKey"/> is set — a shared SQL-Server
-    /// hub, or the pull-only gRPC hub whose half is chosen by <paramref name="role"/>.
+    /// hub, or the pull-only gRPC hub whose half is chosen by <paramref name="role"/>. In gRPC
+    /// mode a central node's store is the SHARED SQL-Server store, and registration fails closed
+    /// when <see cref="HubConnectionStringKey"/> is blank or is a <c>${secret:}</c> reference.
     /// </summary>
     /// <param name="services">The service collection to register into.</param>
     /// <param name="config">Application configuration for options binding.</param>
@@ -192,14 +264,23 @@ public static class SecretsRegistration
         // disagree about whether the hub is on.
         if (UsesGrpcHub(config))
         {
-            // Neither gRPC extension registers the local store — pull-only means local writes
-            // never leave the node, so there is nothing to decorate and the store stays exactly as
-            // configured. Register it first; both halves resolve it.
-            services.AddZbSecrets(config, SecretsSectionPath);
-
             switch (role)
             {
                 case SecretsNodeRole.Central:
+                    // Runs BEFORE any registration so the boot failure names the real problem
+                    // instead of surfacing later as a package validation message or a
+                    // malformed-connection-string fault on first use. On a real boot the Layer-A
+                    // expander in Program.cs has already run the same check even earlier.
+                    EnsureCentralSharedStoreConnectionString(config);
+
+                    // Central's store is the SHARED SQL-Server store: registers
+                    // SqlServerSecretStore as ISecretStore BEFORE calling AddZbSecrets internally,
+                    // displacing SQLite — do NOT also call AddZbSecrets here. One copy of every
+                    // row means both central hubs serve identical manifests by construction, which
+                    // is what makes site-side hub failover between them safe.
+                    services.AddZbSecretsSqlServerStore(
+                        config, SecretsSectionPath, SqlServerSectionPath);
+
                     // Throws when Secrets:GrpcHub:BearerToken is unset. Deliberately not caught: a
                     // hub that starts and refuses every follower looks like a network fault from
                     // the site end, and would be discovered as a stale secret rather than as a
@@ -207,6 +288,13 @@ public static class SecretsRegistration
                     services.AddZbSecretsGrpcHub(config, GrpcHubSectionPath);
                     break;
                 case SecretsNodeRole.Site:
+                    // A site keeps its LOCAL SQLite store — it resolves secrets straight through a
+                    // WAN outage, and sites talk to central, never to central's database. Neither
+                    // gRPC extension registers the local store — pull-only means local writes
+                    // never leave the node, so there is nothing to decorate and the store stays
+                    // exactly as configured. Register it first; the sweep resolves it.
+                    services.AddZbSecrets(config, SecretsSectionPath);
+
                     // Throws when Secrets:GrpcHub:Endpoint or :BearerToken is unset. Same reasoning.
                     services.AddZbSecretsGrpcHubClient(config, GrpcHubSectionPath);
                     break;

src/ZB.MOM.WW.ScadaBridge.Host/appsettings.json

@@ -24,14 +24,16 @@
       "SchemaName": "zbsecrets",
       "SyncInterval": "00:00:30",
       "SyncOnStartup": true,
-      "_comment": "Hub connstr must be seeded in the LOCAL store (or supplied via env) - it cannot come from the hub itself. Enable via Secrets:Replication:Enabled + Mode=SqlServer."
+      "_comment": "Two modes read this section. Mode=SqlServer: the hub connstr must be seeded in the LOCAL store (or supplied via env) - it cannot come from the hub itself. Mode=Grpc: REQUIRED on CENTRAL, where it points BOTH central nodes at ONE shared SQL-Server secret store so the pair cannot diverge (scadaproj#4 - an independent store per central node let one hub serve an authenticated EMPTY manifest); it must be a LITERAL or environment value (Secrets__SqlServer__ConnectionString), never a ${secret:} reference - the pre-host expander needs this store to resolve references, so the reference could never resolve (bootstrap circularity; registration rejects it). SITES leave it empty in every mode - sites talk to central, never to central's database."
     },
     "GrpcHub": {
       "_comment": "Read ONLY when Secrets:Replication:Enabled is true AND Mode=Grpc; inert otherwise. ONE section, both halves: CENTRAL reads BearerToken + MaxNamesPerRequest and hosts the hub on its CentralGrpcPort h2c listener (default 8083, alongside CentralControlService); a SITE reads Endpoint + BearerToken + the sweep timings and pulls. Replication is pull-only by wire contract - the proto has no write RPC - so secrets originate at central and a site cannot push. FAIL-CLOSED: unlike SqlServer mode there is no local-only fallback; a missing BearerToken (either role) or Endpoint (site) is a startup failure naming the key.",
       "_bearerToken": "Shared credential every follower presents. Supply it from appsettings or the environment (Secrets__GrpcHub__BearerToken), NOT as a ${secret:...} reference - resolving that reference is what the hub exists to make possible, so it cannot come from the hub. Same rationale and same handling as the mesh pre-shared keys. Never commit a real value here; the empty default below is fail-closed, not open.",
       "BearerToken": "",
-      "_endpoint": "SITE ONLY. Absolute http/https URI of a CENTRAL node's gRPC (h2c) port - the same address family as ScadaBridge:Communication:CentralGrpcEndpoints, e.g. 'http://central-a-host:8083'. NOT via Traefik (HTTP/1 only). NOTE the asymmetry with CentralGrpcEndpoints: that is a LIST and fails over across the central pair, whereas the hub client dials a SINGLE endpoint. A sweep against a downed central-a therefore stalls rather than failing over - which is survivable because the sweep is best-effort (one warning per interval, then retry) and the site keeps serving its full local last-known-good store, but it does mean secrets stop converging until that central node returns.",
+      "_endpoint": "SITE ONLY. Absolute http/https URI of a CENTRAL node's gRPC (h2c) port - the same address family as ScadaBridge:Communication:CentralGrpcEndpoints, e.g. 'http://central-a-host:8083'. NOT via Traefik (HTTP/1 only). With FallbackEndpoints left empty the hub client dials this SINGLE endpoint only: a sweep against a downed central-a then stalls rather than failing over - survivable because the sweep is best-effort (one warning per interval, then retry) and the site keeps serving its full local last-known-good store, but secrets stop converging until that central node returns. Listing the second central under FallbackEndpoints removes that stall.",
       "Endpoint": "",
+      "_fallbackEndpoints": "SITE ONLY. List the SECOND central node's hub endpoint here (same URI form as Endpoint, e.g. 'http://central-b-host:8083') so a sweep fails over across the central pair instead of stalling on a downed primary. Safe ONLY because both central nodes serve ONE shared SQL secret store, so either hub answers with the same manifest - NEVER list an endpoint backed by an independent store: failing over to an emptier hub is a silent convergence stop, the exact defect scadaproj#4 recorded. Empty by default; leaving it empty keeps the pre-0.5.0 single-endpoint behavior byte-identical.",
+      "FallbackEndpoints": [],
       "SyncInterval": "00:00:30",
       "SyncOnStartup": true,
       "CallDeadline": "00:00:30",

tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SecretsReplicationWiringTests.cs

@@ -1,3 +1,4 @@
+using Grpc.Net.Client;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.Configuration;
@@ -6,6 +7,8 @@ using Microsoft.Extensions.Hosting;
 using ZB.MOM.WW.Secrets.Abstractions;
 using ZB.MOM.WW.Secrets.Replication;
 using ZB.MOM.WW.Secrets.Replicator.Grpc;
+using ZB.MOM.WW.Secrets.Replicator.Grpc.DependencyInjection;
+using ZB.MOM.WW.Secrets.Replicator.SqlServer;
 using ZB.MOM.WW.Secrets.Sqlite;
 
 namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
@@ -37,6 +40,27 @@ public class SecretsReplicationWiringTests
     // Syntactically valid, deliberately unreachable. Never dialled by these tests.
     private const string DummyHubEndpoint = "http://unused.invalid:8083";
 
+    // Syntactically valid, deliberately unreachable. Never dialled by these tests.
+    private const string DummyFallbackHubEndpoint = "http://unused-fallback.invalid:8083";
+
+    /// <summary>
+    /// Keyed service key of the <see cref="GrpcChannel"/> dialing fallback endpoint 0 — the 0.5.0
+    /// package contract: the primary channel keeps
+    /// <see cref="SecretsGrpcHubClientExtensions.ChannelServiceKey"/> unchanged and fallback
+    /// <c>i</c> gets <c>:fallback:i</c> appended.
+    /// </summary>
+    private const string FallbackChannelKey =
+        SecretsGrpcHubClientExtensions.ChannelServiceKey + ":fallback:0";
+
+    /// <summary>
+    /// The package-internal reader seam the follower sweep pulls through
+    /// (<c>ISecretsHubReader</c>). Internal to the package, so it is obtained by full name and
+    /// resolved by <see cref="Type"/> — visibility never gates DI resolution, only compile-time
+    /// references.
+    /// </summary>
+    private static readonly Type HubReaderInterface = typeof(SecretsHubAuthInterceptor).Assembly
+        .GetType("ZB.MOM.WW.Secrets.Replicator.Grpc.ISecretsHubReader", throwOnError: true)!;
+
     // Not a credential — a non-blank placeholder, which is all the fail-closed validators check.
     private const string DummyBearerToken = "test-hub-token";
 
@@ -77,6 +101,21 @@ public class SecretsReplicationWiringTests
         && descriptor.ImplementationType?.Assembly.GetName().Name
             == "ZB.MOM.WW.Secrets.Replicator.SqlServer";
 
+    /// <summary>
+    /// Any descriptor whose service OR implementation type lives in the SQL-Server replicator
+    /// assembly. Honest bound: the package also registers <c>ISecretStore</c>/<c>ISecretsStoreMigrator</c>
+    /// through factory lambdas whose service type is the Abstractions interface and whose
+    /// ImplementationType is null — those two descriptors would evade this scan in isolation. The
+    /// site-purity pin still holds because the same extension unconditionally registers three
+    /// concrete types from the target assembly first, which this scan does catch — so the package
+    /// cannot enter the container without tripping it, even though not every individual descriptor
+    /// it adds is individually detectable.
+    /// </summary>
+    private static bool IsFromSqlServerReplicatorPackage(ServiceDescriptor descriptor) =>
+        descriptor.ServiceType.Assembly.GetName().Name == "ZB.MOM.WW.Secrets.Replicator.SqlServer"
+        || descriptor.ImplementationType?.Assembly.GetName().Name
+            == "ZB.MOM.WW.Secrets.Replicator.SqlServer";
+
     /// <summary>
     /// The gRPC hub package's one hosted service — the follower sweep. Asserted by assembly and
     /// type NAME because <c>GrpcSecretSyncService</c> is internal to the package. The hub SERVER
@@ -87,11 +126,17 @@ public class SecretsReplicationWiringTests
         service.GetType().Assembly.GetName().Name == "ZB.MOM.WW.Secrets.Replicator.Grpc"
         && service.GetType().Name == "GrpcSecretSyncService";
 
+    /// <summary>
+    /// The complete central shape for gRPC mode, connection string included: central's store in
+    /// this mode is the SHARED SQL-Server store (scadaproj#4), so a central config without a
+    /// connection string is not a valid variant — it is a registration failure, pinned separately.
+    /// </summary>
     private static (string Key, string Value)[] GrpcCentralConfig() =>
     [
         ("Secrets:Replication:Enabled", "true"),
         ("Secrets:Replication:Mode", "Grpc"),
         ("Secrets:GrpcHub:BearerToken", DummyBearerToken),
+        ("Secrets:SqlServer:ConnectionString", DummyConnectionString),
     ];
 
     private static (string Key, string Value)[] GrpcSiteConfig() =>
@@ -264,8 +309,10 @@ public class SecretsReplicationWiringTests
         Assert.NotNull(provider.GetService<SecretsHubAuthInterceptor>());
         Assert.True(SecretsRegistration.UsesGrpcHub(config));
 
-        // The local store is still the plain one — pull-only means nothing decorates it.
-        Assert.IsType<SqliteSecretStore>(provider.GetRequiredService<ISecretStore>());
+        // Central's store is the SHARED SQL-Server store — one copy of every row, so both
+        // central hubs serve identical manifests by construction (scadaproj#4). Undecorated:
+        // pull-only means nothing publishes, so there is still nothing to wrap it in.
+        Assert.IsType<SqlServerSecretStore>(provider.GetRequiredService<ISecretStore>());
     }
 
     [Fact]
@@ -302,6 +349,44 @@ public class SecretsReplicationWiringTests
         Assert.Null(provider.GetService<SecretsHubAuthInterceptor>());
     }
 
+    /// <summary>
+    /// A configured fallback endpoint swaps the sweep's reader for the package's failover
+    /// composition and adds one keyed channel per fallback. Safe ONLY because both central nodes
+    /// serve one shared SQL store (scadaproj#4) — the type-name assertion is deliberate: the
+    /// failover reader is internal to the package, and its name is the observable contract here.
+    /// </summary>
+    [Fact]
+    public void GrpcMode_Site_WithFallbackEndpoint_ResolvesTheFailoverReader()
+    {
+        IConfiguration config = BuildConfig(
+        [
+            .. GrpcSiteConfig(),
+            ("Secrets:GrpcHub:FallbackEndpoints:0", DummyFallbackHubEndpoint),
+        ]);
+
+        using ServiceProvider provider = BuildProvider(config, SecretsNodeRole.Site);
+
+        object reader = provider.GetRequiredService(HubReaderInterface);
+        Assert.Equal("FailoverSecretsHubReader", reader.GetType().Name);
+        Assert.NotNull(provider.GetKeyedService<GrpcChannel>(FallbackChannelKey));
+    }
+
+    /// <summary>
+    /// Zero fallbacks pins the unchanged default: the reader stays the plain single-endpoint
+    /// client and no fallback channel enters the container — exactly the pre-0.5.0 shape every
+    /// existing deployment is in.
+    /// </summary>
+    [Fact]
+    public void GrpcMode_Site_WithoutFallbackEndpoints_KeepsThePlainHubClient()
+    {
+        using ServiceProvider provider =
+            BuildProvider(BuildConfig(GrpcSiteConfig()), SecretsNodeRole.Site);
+
+        object reader = provider.GetRequiredService(HubReaderInterface);
+        Assert.Equal("GrpcSecretsHubClient", reader.GetType().Name);
+        Assert.Null(provider.GetKeyedService<GrpcChannel>(FallbackChannelKey));
+    }
+
     // ---------------------------------------------------------------------------------------
     // gRPC mode fails closed. There is no local-only fallback here, unlike SQL-Server mode:
     // a site quietly serving secrets that never converge is the outcome the hub exists to prevent.
@@ -313,9 +398,12 @@ public class SecretsReplicationWiringTests
         var services = new ServiceCollection();
         services.AddLogging();
 
+        // Connection string present so the missing bearer token is the ONE defect in this config —
+        // the connection-string pre-check runs first and would otherwise mask it.
         IConfiguration config = BuildConfig(
             ("Secrets:Replication:Enabled", "true"),
-            ("Secrets:Replication:Mode", "Grpc"));
+            ("Secrets:Replication:Mode", "Grpc"),
+            ("Secrets:SqlServer:ConnectionString", DummyConnectionString));
 
         var ex = Assert.Throws<InvalidOperationException>(
             () => services.AddScadaBridgeSecrets(config, SecretsNodeRole.Central));
@@ -324,6 +412,91 @@ public class SecretsReplicationWiringTests
         Assert.Contains("Secrets:GrpcHub:BearerToken", ex.Message, StringComparison.Ordinal);
     }
 
+    /// <summary>
+    /// Central + gRPC + no connection string is the scadaproj#4 defect asked for by name: each
+    /// central node on an independent local store, one of them answering authenticated followers
+    /// with an EMPTY manifest. It must be a boot failure naming the key, never a quiet fallback.
+    /// </summary>
+    [Theory]
+    [InlineData(null)]
+    [InlineData("")]
+    [InlineData("   ")]
+    public void GrpcMode_Central_WithBlankConnectionString_FailsAtRegistration(string? connectionString)
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+
+        (string, string)[] overrides = connectionString is null
+            ?
+            [
+                ("Secrets:Replication:Enabled", "true"),
+                ("Secrets:Replication:Mode", "Grpc"),
+                ("Secrets:GrpcHub:BearerToken", DummyBearerToken),
+            ]
+            :
+            [
+                ("Secrets:Replication:Enabled", "true"),
+                ("Secrets:Replication:Mode", "Grpc"),
+                ("Secrets:GrpcHub:BearerToken", DummyBearerToken),
+                ("Secrets:SqlServer:ConnectionString", connectionString),
+            ];
+
+        var ex = Assert.Throws<InvalidOperationException>(
+            () => services.AddScadaBridgeSecrets(BuildConfig(overrides), SecretsNodeRole.Central));
+
+        Assert.Contains(
+            SecretsRegistration.HubConnectionStringKey, ex.Message, StringComparison.Ordinal);
+    }
+
+    /// <summary>
+    /// The store's own connection string can never be a <c>${secret:}</c> reference — the pre-host
+    /// expander needs the store this string points at in order to resolve it (bootstrap
+    /// circularity, the same rule as the hub bearer token). Rejected by name rather than left to
+    /// fail as a malformed connection string on first use.
+    /// </summary>
+    [Fact]
+    public void GrpcMode_Central_WithSecretReferenceConnectionString_FailsAtRegistration()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+
+        IConfiguration config = BuildConfig(
+            ("Secrets:Replication:Enabled", "true"),
+            ("Secrets:Replication:Mode", "Grpc"),
+            ("Secrets:GrpcHub:BearerToken", DummyBearerToken),
+            ("Secrets:SqlServer:ConnectionString", "${secret:hub-connection-string}"));
+
+        var ex = Assert.Throws<InvalidOperationException>(
+            () => services.AddScadaBridgeSecrets(config, SecretsNodeRole.Central));
+
+        Assert.Contains(
+            SecretsRegistration.HubConnectionStringKey, ex.Message, StringComparison.Ordinal);
+        Assert.Contains("${secret:", ex.Message, StringComparison.Ordinal);
+        // The reference NAME is config, not secret material — but the message must still not echo
+        // the configured value, because a mistyped value here could be a pasted real credential.
+        Assert.DoesNotContain("hub-connection-string", ex.Message, StringComparison.Ordinal);
+    }
+
+    /// <summary>
+    /// The sites-don't-talk-to-SQL pin: a site node's container must contain NOTHING from the
+    /// SQL-Server replicator package — not just resolve a SQLite store, but hold no descriptor
+    /// that could ever open a connection to central's database.
+    /// </summary>
+    [Fact]
+    public void GrpcMode_Site_HasNoSqlServerPackageTypesInTheContainer()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddScadaBridgeSecrets(BuildConfig(GrpcSiteConfig()), SecretsNodeRole.Site);
+
+        Assert.DoesNotContain(services, IsFromSqlServerReplicatorPackage);
+
+        using ServiceProvider provider = services.BuildServiceProvider(validateScopes: true);
+
+        Assert.IsType<SqliteSecretStore>(provider.GetRequiredService<ISecretStore>());
+        Assert.Null(provider.GetService<SqlServerSecretStore>());
+    }
+
     [Fact]
     public void GrpcMode_Site_WithoutEndpoint_FailsAtRegistration()
     {
@@ -468,13 +641,19 @@ public class SecretsHubEndpointMappingTests
 
     private static bool HasHubEndpoint(WebApplication app) => HubRoutes(app).Count > 0;
 
+    // Syntactically valid, deliberately unreachable — central + gRPC requires the shared
+    // SQL-Server store's connection string at registration; nothing here ever connects.
+    private const string DummyConnectionString =
+        "Server=unused;Database=x;Integrated Security=true;";
+
     [Fact]
     public void GrpcMode_Central_MapsTheHubEndpoint()
     {
         using WebApplication app = BuildCentralApp(
             ("Secrets:Replication:Enabled", "true"),
             ("Secrets:Replication:Mode", "Grpc"),
-            ("Secrets:GrpcHub:BearerToken", DummyBearerToken));
+            ("Secrets:GrpcHub:BearerToken", DummyBearerToken),
+            ("Secrets:SqlServer:ConnectionString", DummyConnectionString));
 
         Assert.True(HasHubEndpoint(app));
     }
@@ -491,7 +670,8 @@ public class SecretsHubEndpointMappingTests
         using WebApplication app = BuildCentralApp(
             ("Secrets:Replication:Enabled", "true"),
             ("Secrets:Replication:Mode", "Grpc"),
-            ("Secrets:GrpcHub:BearerToken", DummyBearerToken));
+            ("Secrets:GrpcHub:BearerToken", DummyBearerToken),
+            ("Secrets:SqlServer:ConnectionString", DummyConnectionString));
 
         Assert.Equal(
             [HubRoutePrefix + "GetManifest", HubRoutePrefix + "GetSecrets"],