feat(host): inbound API key store provider — SqlServer shares one zbauth row set across the central pair; one-shot SQLite import at boot (file renamed .imported, one retry on a concurrent-peer PK race, failure never blocks boot)

Claude-Session: https://claude.ai/code/session_013L3XdmVii9i5sY4FChU8pB

src/ZB.MOM.WW.ScadaBridge.Host/InboundApiKeyStoreImportService.cs

@@ -0,0 +1,165 @@
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
+using ZB.MOM.WW.Auth.ApiKeys.SqlServer;
+
+namespace ZB.MOM.WW.ScadaBridge.Host;
+
+/// <summary>
+/// Seam over <see cref="SqliteApiKeyStoreImporter"/> — the library type is sealed and has no
+/// interface, so this is what lets <see cref="InboundApiKeyStoreImportService"/> be unit-tested
+/// without a SQL Server.
+/// </summary>
+internal interface IInboundApiKeyStoreImporter
+{
+    /// <summary>Copies every key and audit row from the SQLite file at <paramref name="sqlitePath"/> into the shared store.</summary>
+    /// <param name="sqlitePath">Path of the per-node SQLite API-key file.</param>
+    /// <param name="ct">A token to cancel the import.</param>
+    /// <returns>Per-category insert/skip counts.</returns>
+    Task<ApiKeyImportResult> ImportAsync(string sqlitePath, CancellationToken ct);
+}
+
+/// <summary>The production seam implementation: a straight delegation to the library importer.</summary>
+/// <param name="inner">The library importer, resolved from the SQL Server API-key registration.</param>
+internal sealed class LibraryInboundApiKeyStoreImporter(SqliteApiKeyStoreImporter inner)
+    : IInboundApiKeyStoreImporter
+{
+    /// <inheritdoc/>
+    public Task<ApiKeyImportResult> ImportAsync(string sqlitePath, CancellationToken ct)
+        => inner.ImportAsync(sqlitePath, ct);
+}
+
+/// <summary>
+/// One-shot migration of a central node's legacy PER-NODE inbound API-key SQLite file into the
+/// shared SQL Server store, run at boot when
+/// <c>ScadaBridge:InboundApi:ApiKeyStore:Provider = SqlServer</c> (design 2026-09-02, deferred-work
+/// register row 108). Before the shared store existed, a key minted through the management API was
+/// written only to whichever central node served that call, so a failover left every inbound client
+/// 401ing; importing both nodes' files into the one shared store is what closes that gap for keys
+/// that already exist.
+/// </summary>
+/// <remarks>
+///     <para>
+///         The source is <see cref="InboundApiKeyStoreOptions.ImportFromSqlitePath"/> when set,
+///         otherwise the library's own <see cref="ApiKeyOptions.SqlitePath"/> — which still points
+///         at the node's pre-existing file even under the SQL Server provider, since
+///         <c>AddZbApiKeyAuthSqlServer</c> keeps that option bound. A missing file is the normal
+///         steady state and costs nothing.
+///     </para>
+///     <para>
+///         The import itself is idempotent (insert-where-absent per key and per audit row), so both
+///         nodes of a pair may run it and the shared store ends up holding the union of their files.
+///         Both booting at once can still race under read-committed and produce a primary-key
+///         violation (SQL Server error <c>2627</c>) — that ONE error is retried once, because on the
+///         second pass the peer's rows are simply seen and skipped. Any other failure is not retried.
+///     </para>
+///     <para>
+///         On success the file is renamed <c>*.imported</c> (with its <c>-wal</c>/<c>-shm</c>
+///         siblings) so the next boot does no work. Overwriting a stale <c>*.imported</c> is
+///         deliberate: the newer file is the one that was just imported, and keeping the older copy
+///         would make the next boot's rename fail forever.
+///     </para>
+///     <para>
+///         <b>No failure here takes the node down.</b> Every exception — import or rename — is logged
+///         at Error and swallowed: keys already in the shared store keep authenticating, and the
+///         file is left in place so the next boot retries. A host that refused to start on a
+///         half-reachable database would turn a recoverable data-migration problem into an outage.
+///     </para>
+///     <para>
+///         Registered AFTER <c>AddZbApiKeyAuthSqlServer</c> in <c>Program.cs</c> so it starts after
+///         the library's own migration hosted service (hosted services start in registration order)
+///         and the <c>zbauth</c> schema therefore exists before the first insert.
+///     </para>
+/// </remarks>
+/// <param name="importer">The SQLite → shared-store copier.</param>
+/// <param name="storeOptions">Host-side key-store selector (provider + optional explicit import path).</param>
+/// <param name="apiKeyOptions">The library's API-key options, whose <c>SqlitePath</c> is the default source.</param>
+/// <param name="logger">Logger for the import outcome.</param>
+internal sealed class InboundApiKeyStoreImportService(
+    IInboundApiKeyStoreImporter importer,
+    IOptions<InboundApiKeyStoreOptions> storeOptions,
+    IOptions<ApiKeyOptions> apiKeyOptions,
+    ILogger<InboundApiKeyStoreImportService> logger) : IHostedService
+{
+    /// <summary>SQL Server "Violation of PRIMARY KEY constraint" — the only error worth one retry.</summary>
+    private const int PrimaryKeyViolation = 2627;
+
+    /// <inheritdoc/>
+    public async Task StartAsync(CancellationToken cancellationToken)
+    {
+        var configured = storeOptions.Value.ImportFromSqlitePath;
+        var path = string.IsNullOrWhiteSpace(configured) ? apiKeyOptions.Value.SqlitePath : configured;
+
+        if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
+        {
+            logger.LogDebug(
+                "Inbound API key store: no SQLite key store to import (path {Path}).",
+                string.IsNullOrWhiteSpace(path) ? "<unset>" : path);
+            return;
+        }
+
+        try
+        {
+            var result = await ImportWithOneRetryAsync(path, cancellationToken).ConfigureAwait(false);
+
+            logger.LogInformation(
+                "Inbound API key store: imported {Path} into the shared SQL Server store — " +
+                "{KeysInserted} keys inserted, {KeysSkipped} keys already present, " +
+                "{AuditInserted} audit rows inserted, {AuditSkipped} audit rows already present, " +
+                "source found: {SourceFound}.",
+                path, result.KeysInserted, result.KeysSkipped,
+                result.AuditInserted, result.AuditSkipped, result.SourceFound);
+
+            RenameImported(path);
+        }
+        catch (Exception ex)
+        {
+            // Deliberately broad and deliberately not rethrown — see the class remarks.
+            logger.LogError(
+                ex,
+                "Inbound API key store: importing {Path} into the shared SQL Server store failed. " +
+                "Keys already in the shared store keep working; the file is left in place and the " +
+                "next boot will retry.",
+                path);
+        }
+    }
+
+    /// <inheritdoc/>
+    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+
+    private async Task<ApiKeyImportResult> ImportWithOneRetryAsync(string path, CancellationToken ct)
+    {
+        try
+        {
+            return await importer.ImportAsync(path, ct).ConfigureAwait(false);
+        }
+        catch (SqlException ex) when (ex.Number == PrimaryKeyViolation)
+        {
+            logger.LogWarning(
+                ex,
+                "Inbound API key store: importing {Path} hit a primary-key violation — the peer node " +
+                "is most likely importing its own file concurrently. Retrying once; the second pass " +
+                "sees the peer's rows and skips them.",
+                path);
+
+            return await importer.ImportAsync(path, ct).ConfigureAwait(false);
+        }
+    }
+
+    /// <summary>
+    /// Renames the imported file and its WAL/SHM siblings to <c>*.imported</c> so the next boot is a
+    /// no-op, overwriting any stale copy left by an earlier run.
+    /// </summary>
+    /// <param name="path">The file that was just imported.</param>
+    private static void RenameImported(string path)
+    {
+        foreach (var suffix in new[] { string.Empty, "-wal", "-shm" })
+        {
+            var source = path + suffix;
+            if (File.Exists(source))
+            {
+                File.Move(source, source + ".imported", overwrite: true);
+            }
+        }
+    }
+}

src/ZB.MOM.WW.ScadaBridge.Host/Program.cs

@@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.Auth.ApiKeys.DependencyInjection;
+using ZB.MOM.WW.Auth.ApiKeys.SqlServer.DependencyInjection;
 using ZB.MOM.WW.Auth.AspNetCore;
 using ZB.MOM.WW.Health;
 using ZB.MOM.WW.Health.Akka;
@@ -434,6 +435,14 @@ try
         //     value already supplied via appsettings/env wins (AddInMemoryCollection
         //     is registered last, but only fills keys the operator did not set
         //     because we read the existing value first).
+        //
+        // Which BACKEND holds those keys is chosen by
+        // ScadaBridge:InboundApi:ApiKeyStore:Provider (InboundApiKeyStoreOptions):
+        // Sqlite (default) is the per-node file described above; SqlServer shares one
+        // row set across the central pair, so a key minted on either node survives a
+        // failover (design 2026-09-02, deferred-work register row 108). SqlitePath is
+        // still bound under SqlServer — it names the legacy per-node file that
+        // InboundApiKeyStoreImportService copies in once.
         const string apiKeyStoreSection = "ScadaBridge:InboundApi:ApiKeyStore";
         var configuredSqlitePath = builder.Configuration[$"{apiKeyStoreSection}:SqlitePath"];
         var apiKeyStoreDefaults = new Dictionary<string, string?>
@@ -447,7 +456,36 @@ try
         };
         builder.Configuration.AddInMemoryCollection(apiKeyStoreDefaults);
 
-        builder.Services.AddZbApiKeyAuth(builder.Configuration, apiKeyStoreSection);
+        var apiKeyStoreProvider = InboundApiKeyStoreOptions.ReadProvider(builder.Configuration);
+        builder.Services.Configure<InboundApiKeyStoreOptions>(
+            builder.Configuration.GetSection(InboundApiKeyStoreOptions.SectionPath));
+        if (apiKeyStoreProvider == InboundApiKeyStoreProvider.SqlServer)
+        {
+            // Shared store for a central pair (design 2026-09-02, register row 108). The connection
+            // string defaults to the configuration database — the pair already shares it — under
+            // the library's own `zbauth` schema, so no new database or secret is needed. It must be
+            // a literal or an env override, NEVER a ${secret:} reference: the secrets store is
+            // itself SQL-backed and resolves before this point, so a secret-backed key-store string
+            // would be circular.
+            const string sqlSection = apiKeyStoreSection + ":SqlServer";
+            if (string.IsNullOrWhiteSpace(builder.Configuration[$"{sqlSection}:ConnectionString"]))
+            {
+                builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
+                {
+                    [$"{sqlSection}:ConnectionString"] = builder.Configuration["ScadaBridge:Database:ConfigurationDb"],
+                });
+            }
+
+            builder.Services.AddZbApiKeyAuthSqlServer(builder.Configuration, apiKeyStoreSection, sqlSection);
+            builder.Services.AddSingleton<IInboundApiKeyStoreImporter, LibraryInboundApiKeyStoreImporter>();
+            // Registered AFTER AddZbApiKeyAuthSqlServer so it starts after the library's migration
+            // hosted service (hosted services start in registration order) — the schema exists first.
+            builder.Services.AddHostedService<InboundApiKeyStoreImportService>();
+        }
+        else
+        {
+            builder.Services.AddZbApiKeyAuth(builder.Configuration, apiKeyStoreSection);
+        }
 
         // Inbound-API key re-arch, additive: expose the library admin facade
         // (ApiKeyAdminCommands) and the app-side management seam (IInboundApiKeyAdmin)
@@ -463,7 +501,7 @@ try
             sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyAdminStore>(),
             sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyAuditStore>(),
             sp.GetRequiredService<ZB.MOM.WW.Auth.ApiKeys.IApiKeyPepperProvider>(),
-            sp.GetRequiredService<ZB.MOM.WW.Auth.ApiKeys.Sqlite.SqliteAuthStoreMigrator>(),
+            sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyStoreMigrator>(),
             TimeProvider.System));
         builder.Services.AddSingleton<
             ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Security.IInboundApiKeyAdmin,