ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs

using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.ApiKeys.DependencyInjection;
using ZB.MOM.WW.Auth.ApiKeys.SqlServer.DependencyInjection;
using ZB.MOM.WW.Auth.AspNetCore;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.Health.EntityFrameworkCore;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.ScadaBridge.AuditLog;
using ZB.MOM.WW.ScadaBridge.CentralUI;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.DeploymentManager;
using ZB.MOM.WW.ScadaBridge.ExternalSystemGateway;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.Host;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
using ZB.MOM.WW.ScadaBridge.Host.Health;
using ZB.MOM.WW.ScadaBridge.InboundAPI;
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
using ZB.MOM.WW.ScadaBridge.KpiHistory;
using ZB.MOM.WW.ScadaBridge.ManagementService;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
using ZB.MOM.WW.ScadaBridge.NotificationService;
using ZB.MOM.WW.ScadaBridge.Security;
using ZB.MOM.WW.ScadaBridge.SiteCallAudit;
using ZB.MOM.WW.ScadaBridge.TemplateEngine;
using ZB.MOM.WW.ScadaBridge.Transport;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Configuration;
using ZB.MOM.WW.Secrets.DependencyInjection;
using ZB.MOM.WW.Secrets.Ui;
using ZB.MOM.WW.Telemetry;
using Serilog;

// Terminate honestly on ANY unhandled exception, from any thread — registered before the
// first statement that can throw. In a container this process runs as PID 1, and Linux
// ignores default-action signals sent to PID 1, so the runtime's crash path (print the
// banner, then abort() → SIGABRT) can never actually kill the process: observed live, it
// printed the banner and then spun the main thread at 100% CPU with the container still
// `running`, so `restart: unless-stopped` never fired and every fail-closed boot refusal
// became a silent manual-restart outage (ScadaBridge#34, reproduced 2026-08-07).
// Environment.Exit uses the exit() syscall, which PID 1 CAN perform, and 134 preserves the
// 128+SIGABRT code the abort would have produced. This never fires under test hosts:
// WebApplicationFactory catches entry-point exceptions itself, so they are never
// "unhandled" there and the designed boot-refusal exceptions still propagate to tests.
// The docker compose additionally sets `init: true` for the crash paths that bypass this
// event entirely (Environment.FailFast, runtime-internal aborts).
AppDomain.CurrentDomain.UnhandledException += (_, e) =>
{
    // Print ourselves: exiting here preempts the runtime's own banner.
    Console.Error.WriteLine(e.ExceptionObject);
    try { Log.CloseAndFlush(); } catch { /* stderr above already has the exception */ }
    Environment.Exit(134);
};

// SCADABRIDGE_CONFIG determines which role-specific config to load (Central or Site)
// DOTNET_ENVIRONMENT/ASPNETCORE_ENVIRONMENT stay as "Development" for dev tooling (static assets, EF migrations, etc.)
var scadabridgeConfig = Environment.GetEnvironmentVariable("SCADABRIDGE_CONFIG")
    ?? Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT")
    ?? "Production";

var configuration = new ConfigurationBuilder()
    .AddJsonFile("appsettings.json", optional: false)
    .AddJsonFile($"appsettings.{scadabridgeConfig}.json", optional: true)
    .AddEnvironmentVariables()
    .AddCommandLine(args)
    .Build();

// Expand ${secret:...} config references before any validator/binder sees them (Layer A).
// Throwaway provider — disposed here, shares no singletons with the host container.
//
// The expander MUST read the SAME store the running node will serve — SecretsRegistration.
// UsesCentralSharedStore is the single predicate deciding that, shared with the runtime
// registration in AddScadaBridgeSecrets, so the two can never disagree. On a Central node with
// Secrets:Store:Provider=SqlServer that is the SHARED SQL-Server store: an expander left
// on SQLite there would resolve pre-host ${secret:} references (e.g. ScadaBridge:Security:
// JwtSigningKey) from a store the node does not actually serve, so a value rotated through the
// shared store would never reach the pre-host config, and two central nodes could disagree on
// what a token resolves to. Every other case (sites, the default per-node SQLite store) keeps the
// local SQLite path exactly as it always was. A blank or ${secret:}-valued connection string on
// the shared-store path fails HERE, through the same EnsureCentralSharedStoreConnectionString
// check AddScadaBridgeSecrets runs — this is the earliest code that would otherwise hand the bad
// value to SqlConnection, whose generic "initialization string" format error would bury the
// designed message.
//
// Two consequences worth knowing. This block runs BEFORE Serilog exists and outside the
// try/catch below, so a SQL Server outage at central boot exits with a bare stderr stack
// trace, no structured log — honest and container-restart-retryable, just unenriched; the
// SQLite path had the same window but its only failure mode was a local file. (That exit
// is real only because of the UnhandledException handler at the top of this file — the
// runtime's own crash path wedges as container PID 1, ScadaBridge#34.) And the role read
// stays here at the composition root rather than inside SecretsRegistration: that class
// refuses to read the node role from configuration (a config-read role could silently turn a
// site into a shared-store central — see SecretsNodeRole), while Program.cs must read it to
// branch the whole composition anyway.
var expanderRole = string.Equals(
    configuration["ScadaBridge:Node:Role"], "Central", StringComparison.OrdinalIgnoreCase)
        ? SecretsNodeRole.Central
        : SecretsNodeRole.Site;

// Secrets 0.6.2 validates Secrets:SqlitePath as absolute AND outside the content root, finding the
// content root from IHostEnvironment — which this throwaway ServiceCollection does not have, and
// which does not exist yet anyway (the host is not built until further down). Both the resolution
// and the registration therefore live in SecretsRegistration, where a test can enforce them; see
// AddPreHostSqliteExpander for why the content-root argument is load-bearing and silent when
// dropped, and PreHostSecretsContentRootTests for the regression guard.
var expanderServices = new ServiceCollection();
expanderServices.AddPreHostExpander(
    configuration, expanderRole, SecretsRegistration.ResolveHostContentRoot());

#pragma warning disable ASP0000 // deliberate throwaway container
await using (var secretsProvider = expanderServices.BuildServiceProvider())
#pragma warning restore ASP0000
{
    // ISecretsStoreMigrator is the store-agnostic seam both AddZbSecrets (SQLite) and
    // AddZbSecretsSqlServerStore register — resolving through it means this block does not need
    // to know, or re-derive, which concrete store AddPreHostExpander just composed.
    await secretsProvider.GetRequiredService<ISecretsStoreMigrator>()
        .MigrateAsync(default);

    var resolver = secretsProvider.GetRequiredService<ISecretResolver>();
    await new SecretReferenceExpander(resolver)
        .ExpandConfigurationAsync(configuration, default);
}

// Full startup validation — fail fast before any DI or actor system setup
StartupValidator.Validate(configuration);

// Bound node options: the single home of port defaults (NodeOptions), so the Kestrel setup below
// never repeats literal 8083/8084 defaults at the call sites (review 2026-10-21 08/O5).
var nodeOptions = configuration.GetSection("ScadaBridge:Node").Get<ZB.MOM.WW.ScadaBridge.Host.NodeOptions>()
    ?? new ZB.MOM.WW.ScadaBridge.Host.NodeOptions();

// Read node options for Serilog enrichment
var nodeRole = configuration["ScadaBridge:Node:Role"]!;
var nodeHostname = configuration["ScadaBridge:Node:NodeHostname"] ?? "unknown";
var siteId = configuration["ScadaBridge:Node:SiteId"] ?? "central";

// Serilog structured logging.
// Minimum level is driven by ScadaBridge:Logging:MinimumLevel (LoggingOptions).
// Console and file sinks are defined in the `Serilog` configuration
// section (appsettings.json) and applied via ReadFrom.Configuration inside the
// factory — the sink set, output template, file path and rolling interval are all
// configuration-driven, not hard-coded here.
Log.Logger = ZB.MOM.WW.ScadaBridge.Host.LoggerConfigurationFactory
    .Build(configuration, nodeRole, siteId, nodeHostname)
    .CreateLogger();

// The host's ApplicationStopping token, captured once the host is built. A normal Windows service
// stop (or SIGTERM) cancels it BEFORE RunAsync unwinds, and WindowsServiceLifetime.StopAsync can
// then surface an OperationCanceledException out of RunAsync after shutdown has already finished.
// The catch below uses it to tell that routine stop from a genuine fatal exit (ScadaBridge#76).
var hostStopping = CancellationToken.None;

try
{
    Log.Information("Starting ScadaBridge host as {Role} on {Hostname}", nodeRole, nodeHostname);

    // Non-fatal preflight findings (StartupValidator.CollectWarnings) are logged here rather than
    // inside Validate because the validator runs before Serilog exists. Today: a clustered Central
    // still on the per-node SQLite inbound API key store (design 2026-09-02, register row 108).
    foreach (var warning in StartupValidator.CollectWarnings(configuration))
        Log.Warning("Startup configuration warning: {Warning}", warning);

    if (nodeRole.Equals("Central", StringComparison.OrdinalIgnoreCase))
    {
        var builder = WebApplication.CreateBuilder(args);
        builder.Configuration.AddConfiguration(configuration);

        // Captive-dependency validation (deferred register row 50). See the identical
        // block in the Site branch below and ServiceProviderValidation for the full rationale.
        builder.Host.UseDefaultServiceProvider(ServiceProviderValidation.Configure);

        // Serilog
        builder.Host.UseSerilog();

        // Windows Service support (no-op when not running as a Windows Service)
        builder.Host.UseWindowsService();

        // Explicit Kestrel h2c listener for the central-hosted gRPC control plane
        // (CentralControlService): HTTP/2-only, on its own port (default 8083, symmetric
        // with the site GrpcPort — a node is either central or a site, never both). gRPC
        // does NOT go through Traefik (HTTP/1 only); sites reach this port by container name.
        //
        // WARNING — the trap the rig caught (T1A.2 shipped it, fixed here): calling
        // options.Listen*/ListenAnyIP puts Kestrel into EXPLICIT-ENDPOINTS mode, which
        // SUPPRESSES the URLs from ASPNETCORE_URLS/--urls entirely — it is NOT additive.
        // Central's ENTIRE HTTP/1 surface (Central UI, Management + Inbound API, and the
        // /health/* endpoints Traefik + IActiveNodeGate depend on) lives on that URL
        // (http://+:5000 on the rig, a different port in production). If we bind only the
        // gRPC port, :5000 vanishes and central is reachable over gRPC while the UI, the
        // management API and health checks are all dead — with no startup error. Unit tests
        // use TestServer and never bind real Kestrel, so only a live node exposes this.
        // The site branch has the same ConfigureKestrel shape but no ASPNETCORE_URLS surface
        // to lose (it binds every port it needs — gRPC + metrics — explicitly). Central must
        // therefore RE-BIND its HTTP port(s) here alongside the gRPC port.
        var centralGrpcPort = nodeOptions.CentralGrpcPort;
        // "urls" is WebHostDefaults.ServerUrlsKey — the host setting ASPNETCORE_URLS/--urls
        // populate. Read it as a literal so this needs no extra Hosting using.
        var httpUrls = configuration["ASPNETCORE_URLS"]
            ?? builder.WebHost.GetSetting("urls")
            ?? "http://+:5000";
        var httpEndpoints = ParseHttpBindEndpoints(httpUrls);
        var httpPorts = httpEndpoints.Select(e => e.Port).ToList();
        // ScadaBridge#108: an https URL needs a configured node certificate; without one the
        // node refuses to boot rather than silently serving plaintext on that port.
        var httpsCertificate = CentralHttpBinding.LoadCertificateIfNeeded(httpEndpoints, nodeOptions.Tls);

        // Register row 56: central's dedicated Prometheus listener, OPT-IN.
        //
        // /metrics is already mapped on central (app.MapZbMetrics() below) and, because it
        // rides endpoint routing, it is served on every listener this node binds — so it has
        // always been reachable on the HTTP surface (:5000 on the rig). What central lacked
        // was the site branch's dedicated MetricsPort, which is what the established
        // live-scrape recipe probes (`docker exec <node> bash -c '</dev/tcp/localhost/8084'`).
        // A probe of central on :8084 therefore failed, and central-only series (KPI/outbox
        // counters, the import-session gauge + rejection counter, alarm-cache gauges) read as
        // unscrapeable even though they were being exported.
        //
        // Read as a PRESENCE check, not GetValue's defaulted read, deliberately: NodeOptions
        // defaults MetricsPort to 8084, so a defaulted read would make every existing central
        // node start binding a new port on upgrade — and a port already in use is a hard boot
        // failure, not a degraded one. Central gains the listener only where an operator asked
        // for it. Site nodes are unaffected; they still bind theirs unconditionally.
        var centralMetricsPort = int.TryParse(configuration["ScadaBridge:Node:MetricsPort"], out var mp)
            ? mp
            : (int?)null;

        builder.WebHost.ConfigureKestrel(options =>
        {
            options.Limits.Http2.KeepAlivePingDelay = SiteKestrelSetup.Http2KeepAlivePingDelay;
            options.Limits.Http2.KeepAlivePingTimeout = SiteKestrelSetup.Http2KeepAlivePingTimeout;
            // The HTTP/1.1 (+ HTTP/2) surface from the configured URLs, re-declared so it
            // survives the explicit-endpoints switch above.
            CentralHttpBinding.BindHttpSurface(options, httpEndpoints, httpsCertificate);

            // The gRPC control plane, HTTP/2 h2c only, on its own port.
            options.ListenAnyIP(centralGrpcPort, listenOptions =>
            {
                listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2;
            });

            // The Prometheus scrape listener (register row 56), when configured. Skipped
            // when it would double-bind a port already declared above — Kestrel throws on a
            // duplicate ListenAnyIP, and /metrics is reachable on that port either way since
            // it rides endpoint routing. HTTP/1.1-capable on purpose: a standard Prometheus
            // scraper cannot negotiate h2c, which is why this cannot share the gRPC port.
            if (centralMetricsPort is { } cmp
                && cmp != centralGrpcPort
                && !httpPorts.Contains(cmp))
            {
                options.ListenAnyIP(cmp, listenOptions =>
                {
                    listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http1AndHttp2;
                });
            }
        });

        // Shared components
        builder.Services.AddClusterInfrastructure();
        builder.Services.AddCommunication();
        // Central-only communication services (ReconcileService — scoped, holds the
        // central-only IDeploymentManagerRepository). Split out of AddCommunication by
        // deferred register row 50: registering it on a site node produced an
        // unconstructible descriptor that failed ValidateOnBuild for the whole site graph.
        builder.Services.AddCentralCommunication();

        // Per-site gRPC preshared keys. Central-only: it is the side that dials sites, and
        // the only side whose key set is dynamic (sites come from the configuration
        // database, so there is no fixed list of ${secret:} references to expand at boot —
        // hence a runtime resolver rather than the pre-host SecretReferenceExpander a site
        // node uses for its single key). Registered before the clients that consume it.
        builder.Services.AddSingleton<
            ZB.MOM.WW.ScadaBridge.Communication.Grpc.ISitePskProvider, SitePskProvider>();

        // Central-hosted gRPC control plane (T1A.2). CentralControlAuthInterceptor is the
        // per-site sibling of the site's ControlPlaneAuthInterceptor: it verifies the Bearer
        // token against the key for the site named in the required x-scadabridge-site header,
        // resolved through the ISitePskProvider registered just above. Registered BY TYPE on
        // AddGrpc exactly as the site branch registers its interceptors — NOT as a DI singleton,
        // which would let DI hand the instance back and bypass Grpc.AspNetCore's own activation
        // path (the shape that once hid a two-public-constructor defect until the rig caught it).
        // CentralControlGrpcService decodes each request onto the same in-process message the
        // removed ClusterClient path used to carry, and Asks CentralCommunicationActor — zero handler logic here.
        builder.Services.AddGrpc(options =>
        {
            options.MaxReceiveMessageSize = ZB.MOM.WW.ScadaBridge.Communication.Grpc.GrpcMessageLimits.MaxMessageBytes;
            options.MaxSendMessageSize = ZB.MOM.WW.ScadaBridge.Communication.Grpc.GrpcMessageLimits.MaxMessageBytes;
            options.Interceptors.Add<CentralControlAuthInterceptor>();
        });
        builder.Services.AddSingleton<
            ZB.MOM.WW.ScadaBridge.Communication.Grpc.CentralControlGrpcService>();

        // Shared per-site gRPC channel-pair provider (sticky failover/failback) backing the
        // GrpcSiteTransport when ScadaBridge:Communication:SiteTransport=Grpc. Central-only, and a
        // no-op until CentralCommunicationActor builds the gRPC transport (its failback loop starts
        // on first use), so registering it unconditionally is harmless under the default Akka path.
        builder.Services.AddSingleton<
            ZB.MOM.WW.ScadaBridge.Communication.Grpc.SitePairChannelProvider>();

        builder.Services.AddHealthMonitoring();
        builder.Services.AddCentralHealthAggregation();
        // Central resolves definitions from SQL Server on every call; cache them briefly
        // (review 2026-10-21 05/P2). Registered BEFORE AddExternalSystemGateway so an explicit
        // ScadaBridge:ExternalSystemGateway:DefinitionCacheTtl binding still wins. Sites keep it
        // off — their repository is already a deploy-invalidated snapshot.
        builder.Services.Configure<ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.ExternalSystemGatewayOptions>(
            o => o.DefinitionCacheTtl = TimeSpan.FromSeconds(10));
        builder.Services.AddExternalSystemGateway();
        builder.Services.AddNotificationService();

        // Central-only components
        // Notification Outbox: central owns SMTP delivery; the Email adapter reuses the
        // AddNotificationService() SMTP machinery above. AddNotificationOutbox binds
        // NotificationOutboxOptions via BindConfiguration, so no explicit Configure is needed.
        builder.Services.AddNotificationOutbox();
        // Transport — central-only bundle export/import pipeline. Binds
        // TransportOptions from ScadaBridge:Transport via BindConfiguration; no
        // explicit Configure needed.
        builder.Services.AddTransport();
        // Script-artifact invalidation bus: in-process, per-node pub/sub
        // for ScriptArtifactsChanged. The Transport bundle importer publishes
        // post-commit; the Inbound API's ScriptArtifactChangeSubscriber (a hosted
        // service registered by AddInboundAPI below) consumes ApiMethod notifications
        // and invalidates the compiled-handler cache (the
        // per-request revision check remains the correctness fallback).
        // SharedScript/Template notifications currently have NO consumer — nothing at
        // central caches compiled artifacts of those kinds. Central-only.
        builder.Services.AddSingleton<
            ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Scripting.IScriptArtifactChangeBus,
            ZB.MOM.WW.ScadaBridge.Host.Services.InProcessScriptArtifactChangeBus>();
        // Audit Log — central node owns the AuditLogIngestActor singleton +
        // IAuditLogRepository. The site writer chain is still registered (lazy
        // singletons) but is never resolved on a central node.
        builder.Services.AddAuditLog(builder.Configuration);
        // Central-only hosted service that rolls
        // pf_AuditLog_Month forward monthly. Depends on IPartitionMaintenance
        // (registered below by AddConfigurationDatabase).
        builder.Services.AddAuditLogCentralMaintenance(builder.Configuration);
        // Central-only registration backing the two
        // maintenance singletons started in AkkaHostedService: the production
        // ISiteEnumerator + IPullAuditEventsClient (gRPC) used by the
        // SiteAuditReconciliationActor, plus the AuditLogPurgeOptions /
        // SiteAuditReconciliationOptions bindings consumed by both singletons.
        // Central-only by design (it dials sites), kept out of AddAuditLog.
        builder.Services.AddAuditLogCentralReconciliationClient(builder.Configuration);
        // Site Call Audit — central node owns the SiteCallAuditActor
        // singleton. The extension itself currently registers
        // nothing — actor Props are constructed inline in AkkaHostedService —
        // but the call is here for symmetry with the other audit composition
        // roots so future per-actor DI lands without touching Program.cs.
        builder.Services.AddSiteCallAudit();
        // KPI History — central-only. Binds KpiHistoryOptions from
        // ScadaBridge:KpiHistory and registers the validated options consumed by
        // the KpiHistoryRecorderActor cluster singleton (started in
        // AkkaHostedService). Observability/best-effort: NOT readiness-gated.
        builder.Services.AddKpiHistory(builder.Configuration);
        // Cross-validates the rate-fold interval triple
        // (KpiHistoryOptions.SampleInterval, NotificationOutboxOptions.DeliveredKpiWindow,
        // SiteCallAuditOptions.KpiInterval) at boot. Rides KpiHistoryOptions' existing
        // ValidateOnStart trigger set by AddKpiHistory above.
        builder.Services.TryAddEnumerable(
            ServiceDescriptor.Singleton<IValidateOptions<KpiHistoryOptions>, KpiRateFoldIntervalCrossValidator>());
        // The Audit Log's IKpiSampleSource, enumerated by the recorder registered just
        // above. Central-only (its IAuditLogRepository is central-only); split out of
        // AddAuditLog by deferred register row 50 for the same reason as
        // AddCentralCommunication above.
        builder.Services.AddAuditLogCentralKpi();
        builder.Services.AddTemplateEngine();
        builder.Services.AddDeploymentManager();
        // Host is the composition root and owns config-coupled wiring: register the
        // shared LDAP auth (binds LdapOptions + IValidateOptions<LdapOptions> with
        // ValidateOnStart + ILdapAuthService singleton) here, then AddSecurity() for the
        // config-free remainder. AddSecurity is a component library and takes no
        // IConfiguration (Options pattern only). Behaviour-preserving: identical
        // registrations to the previous AddSecurity(builder.Configuration) call.
        builder.Services.AddZbLdapAuth(
            builder.Configuration,
            ZB.MOM.WW.ScadaBridge.Security.ServiceCollectionExtensions.LdapSectionPath);
        // Dev disable-login flag (config-coupled, so read + bound here at the composition root,
        // mirroring AddZbLdapAuth). Default false. See AuthDisableLoginOptions / disable-login design doc.
        builder.Services.AddOptions<ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions>()
            .Bind(builder.Configuration.GetSection(
                ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.SectionName));
        var disableLogin = builder.Configuration
            .GetSection(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.SectionName)
            .GetValue<bool>(nameof(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.DisableLogin));
        // Fail-fast guard: DisableLogin outside Development requires the explicit
        // AllowOutsideDevelopment acknowledgement key. See DisableLoginGuard.
        ZB.MOM.WW.ScadaBridge.Security.Auth.DisableLoginGuard.Validate(
            disableLogin,
            builder.Environment.EnvironmentName,
            builder.Configuration
                .GetSection(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.SectionName)
                .GetValue<bool>(nameof(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.AllowOutsideDevelopment)));
        builder.Services.AddSecurity(disableLogin);
        builder.Services.AddCentralUI();
        // Local SQLite store by default. With Secrets:Store:Provider=SqlServer this central node's
        // store becomes the SHARED SQL-Server database instead — no hub, no follower sweep, just
        // both central nodes reading and writing the same rows, so a clustered pair can never ship
        // sites a drifted secret set. The Layer-A pre-host expander above resolves this
        // same predicate (SecretsRegistration.UsesCentralSharedStore) so it always reads the store
        // this node actually serves. See SecretsRegistration for why the SQL gate exists (eager
        // options validation) and why the connection string can never itself be a ${secret:}
        // reference.
        builder.Services.AddScadaBridgeSecrets(builder.Configuration, SecretsNodeRole.Central);
        // An in-place rotation of a site's SB-GRPC-PSK-{siteId} through /admin/secrets must drop
        // central's cached copy of that key on BOTH central nodes (Gitea #96). Wraps the library's
        // ISecretCacheInvalidator; the peer half rides the central cache-invalidation topic
        // (AddCentralCacheInvalidation, ScadaBridge#100 item 3 — also used by the role-mapping,
        // external-definition and inbound-API caches), wired in AkkaHostedService.RegisterCentralActors.
        builder.Services.AddCentralCacheInvalidation();
        builder.Services.AddSitePskRotationInvalidation();
        // Secrets UI authorization: adds the named policies secrets:manage + secrets:reveal
        // (role-based) consumed by the /admin/secrets page. AddSecretsAuthorization only ADDS
        // these two policies via Configure<AuthorizationOptions> — it composes additively with
        // AddScadaBridgeAuthorization (invoked inside AddSecurity above) without clobbering.
        // The library's AdminRole ("Administrator") satisfies both policies, and ScadaBridge
        // builds identities with RoleClaimType = ZbClaimTypes.Role, so an Administrator is
        // authorized for both manage + reveal. Done Host-side to keep Secrets.Ui out of Security.
        builder.Services.Configure<AuthorizationOptions>(o => o.AddSecretsAuthorization());
        // Secrets.Ui components audit through the SHARED ZB.MOM.WW.Audit.IAuditWriter seam,
        // which is deliberately distinct from the repo's own Commons IAuditWriter — bridge it
        // to the central direct-write path (ICentralAuditWriter → dbo.AuditLog), NOT the site
        // SQLite chain, which is never resolved on central and has no forwarder here
        // (ScadaBridge#22; component activation 500'd without this registration).
        builder.Services.AddSingleton<ZB.MOM.WW.Audit.IAuditWriter>(
            sp => new ZB.MOM.WW.ScadaBridge.Host.Services.CentralSharedSeamAuditWriter(
                sp.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ICentralAuditWriter>()));
        builder.Services.AddInboundAPI();

        // Inbound-API auth re-arch: the shared ZB.MOM.WW.Auth.ApiKeys verifier +
        // SQLite store + startup migration are now the SOLE inbound-API auth path.
        // The POST /api/{methodName} endpoint authenticates Bearer tokens
        // (sbk_<keyId>_<secret>) and authorizes by scope == method name through this
        // verifier. The legacy peppered-HMAC X-API-Key path — the SQL Server ApiKey
        // entity, ApiKeyValidator, and IApiKeyHasher — was retired; the
        // ScadaBridge:InboundApi:ApiKeyPepper config key is now consumed only as the
        // library verifier's pepper secret (PepperSecretName below).
        //
        // ApiKeyOptions is an init-only record, so the contract-mandated values
        // are injected as in-memory configuration UNDER the bound section path
        // (ScadaBridge:InboundApi:ApiKeyStore) rather than mutated post-bind:
        //   - TokenPrefix = "sbk"           (the inbound token prefix)
        //   - PepperSecretName points at the EXISTING inbound-API pepper config key
        //     (reused so one secret backs both the legacy and new path during the
        //     additive window)
        //   - RunMigrationsOnStartup = true (hosted service creates the schema)
        //   - SqlitePath defaults under the content root's data/ directory, but any
        //     value already supplied via appsettings/env wins (AddInMemoryCollection
        //     is registered last, but only fills keys the operator did not set
        //     because we read the existing value first).
        //
        // Which BACKEND holds those keys is chosen by
        // ScadaBridge:InboundApi:ApiKeyStore:Provider (InboundApiKeyStoreOptions):
        // Sqlite (default) is the per-node file described above; SqlServer shares one
        // row set across the central pair, so a key minted on either node survives a
        // failover (design 2026-09-02, deferred-work register row 108). SqlitePath is
        // still bound under SqlServer — it names the legacy per-node file that
        // InboundApiKeyStoreImportService copies in once.
        const string apiKeyStoreSection = "ScadaBridge:InboundApi:ApiKeyStore";
        var configuredSqlitePath = builder.Configuration[$"{apiKeyStoreSection}:SqlitePath"];
        var apiKeyStoreDefaults = new Dictionary<string, string?>
        {
            [$"{apiKeyStoreSection}:TokenPrefix"] = "sbk",
            [$"{apiKeyStoreSection}:PepperSecretName"] = "ScadaBridge:InboundApi:ApiKeyPepper",
            [$"{apiKeyStoreSection}:RunMigrationsOnStartup"] = "true",
            [$"{apiKeyStoreSection}:SqlitePath"] = string.IsNullOrWhiteSpace(configuredSqlitePath)
                ? Path.Combine(builder.Environment.ContentRootPath, "data", "inbound-api-keys.sqlite")
                : configuredSqlitePath,
        };
        builder.Configuration.AddInMemoryCollection(apiKeyStoreDefaults);

        var apiKeyStoreProvider = InboundApiKeyStoreOptions.ReadProvider(builder.Configuration);
        builder.Services.Configure<InboundApiKeyStoreOptions>(
            builder.Configuration.GetSection(InboundApiKeyStoreOptions.SectionPath));
        if (apiKeyStoreProvider == InboundApiKeyStoreProvider.SqlServer)
        {
            // Shared store for a central pair (design 2026-09-02, register row 108). The connection
            // string defaults to the configuration database — the pair already shares it — under
            // the library's own `zbauth` schema, so no new database or secret is needed. It must be
            // a literal or an env override, NEVER a ${secret:} reference: the secrets store is
            // itself SQL-backed and resolves before this point, so a secret-backed key-store string
            // would be circular.
            const string sqlSection = apiKeyStoreSection + ":SqlServer";
            if (string.IsNullOrWhiteSpace(builder.Configuration[$"{sqlSection}:ConnectionString"]))
            {
                builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
                {
                    [$"{sqlSection}:ConnectionString"] = builder.Configuration["ScadaBridge:Database:ConfigurationDb"],
                });
            }

            builder.Services.AddZbApiKeyAuthSqlServer(builder.Configuration, apiKeyStoreSection, sqlSection);
            builder.Services.AddSingleton<IInboundApiKeyStoreImporter, LibraryInboundApiKeyStoreImporter>();
            // Registered AFTER AddZbApiKeyAuthSqlServer so it starts after the library's migration
            // hosted service (hosted services start in registration order) — the schema exists first.
            builder.Services.AddHostedService<InboundApiKeyStoreImportService>();
        }
        else
        {
            builder.Services.AddZbApiKeyAuth(builder.Configuration, apiKeyStoreSection);
        }

        // Inbound-API key re-arch, additive: expose the library admin facade
        // (ApiKeyAdminCommands) and the app-side management seam (IInboundApiKeyAdmin)
        // in the SAME container as AddZbApiKeyAuth, so CLI + CentralUI later create /
        // list / enable / disable / delete inbound keys and edit their method-scopes
        // through one shared path. AddZbApiKeyAuth registers the stores/pepper/migrator
        // but NOT ApiKeyAdminCommands itself, so it is composed here. CentralUI resolves
        // from this same provider (it is registered via AddCentralUI() above), so the
        // seam is reachable from both the ManagementActor and CentralUI pages — exactly
        // as IInboundApiRepository already is.
        builder.Services.AddSingleton(sp => new ZB.MOM.WW.Auth.ApiKeys.Admin.ApiKeyAdminCommands(
            sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyOptions>>().Value,
            sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyAdminStore>(),
            sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyAuditStore>(),
            sp.GetRequiredService<ZB.MOM.WW.Auth.ApiKeys.IApiKeyPepperProvider>(),
            sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyStoreMigrator>(),
            TimeProvider.System));
        builder.Services.AddSingleton<
            ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Security.IInboundApiKeyAdmin,
            LibraryInboundApiKeyAdmin>();

        builder.Services.AddManagementService();

        var configDbConnectionString = configuration["ScadaBridge:Database:ConfigurationDb"]
            ?? throw new InvalidOperationException("ScadaBridge:Database:ConfigurationDb connection string is required for Central role.");
        builder.Services.AddConfigurationDatabase(configDbConnectionString);
        builder.Services.AddOptions<ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.ConfigAuditOptions>()
            .Bind(configuration.GetSection(ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.ConfigAuditOptions.SectionName))
            .ValidateOnStart();

        // Health checks for readiness gating — shared ZB.MOM.WW.Health probes.
        // Ready tier (/health/ready): database + akka-cluster + required-singletons.
        // Active tier (/health/active): active-node = oldest Up member (singleton host) AND
        // database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
        // rotation. The active-node check is the shared ActiveNodeHealthCheck, which selects by age
        // as of Health 0.3.0; through 0.2.1 it selected by leadership, which diverges from singleton
        // placement after a restart and names both sides during a partition, so this host kept a
        // private copy until the shared one was fixed.
        // The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
        // the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
        builder.Services.AddHealthChecks()
            .AddTypeActivatedCheck<DatabaseHealthCheck<ScadaBridgeDbContext>>(
                "database",
                failureStatus: null,
                tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })
            .AddTypeActivatedCheck<AkkaClusterHealthCheck>(
                "akka-cluster",
                failureStatus: null,
                tags: new[] { ZbHealthTags.Ready },
                args: AkkaClusterStatusPolicy.Default)
            // Readiness ALSO reflects "required cluster singletons running"
            // (REQ-HOST-4a). Probes each central singleton's local ClusterSingletonProxy
            // with a bounded Identify and degrades to Unhealthy if any required singleton
            // is unreachable. Registered inside the Central-role branch (this is it) so the
            // check is naturally role-scoped — site nodes never run it. It resolves
            // ActorSystem from DI per probe, like the akka-cluster check above, and is
            // leadership-agnostic so a ready standby still reports ready (the proxy reaches
            // the singleton from either node).
            .AddTypeActivatedCheck<RequiredSingletonsHealthCheck>(
                "required-singletons",
                failureStatus: null,
                tags: new[] { ZbHealthTags.Ready })
            // ScadaBridge#110: Degraded (HTTP 200) while a Transport:Signing:ProductionBoundary
            // cluster still accepts unsigned bundles.
            .AddTypeActivatedCheck<BundleSigningPolicyHealthCheck>(
                "bundle-signing-policy",
                failureStatus: null,
                tags: new[] { ZbHealthTags.Ready })
            // Oldest-Up-member check (review 01 [High]), unscoped: every central member competes for
            // the one active slot. Since ZB.MOM.WW.Health 0.3.0 this IS the shared check — it selects
            // by age, so the host-local OldestNodeActiveHealthCheck that existed only to avoid the old
            // leader-based selection is gone. Resolves ActorSystem from DI, like the akka-cluster
            // check above.
            .AddTypeActivatedCheck<ActiveNodeHealthCheck>(
                "active-node",
                failureStatus: null,
                tags: new[] { ZbHealthTags.Active });

        // Akka.NET bootstrap via hosted service
        builder.Services.AddSingleton<AkkaHostedService>();
        builder.Services.AddHostedService(sp => sp.GetRequiredService<AkkaHostedService>());

        // Bridge the AkkaHostedService-owned ActorSystem to DI as a SINGLETON via
        // GetOrCreateActorSystem(). The shared ZB.MOM.WW.Health Akka checks resolve ActorSystem
        // from DI, per probe, inside a child scope. ActorSystem is IDisposable, so a TRANSIENT
        // (or scoped) bridge is captured-and-disposed by each probe's scope — disposing the live
        // system mid-flight (CoordinatedShutdown/ActorSystemTerminateReason) and wedging the
        // central report pages at the 30s Ask timeout. A singleton is resolved from the root and
        // never disposed by a child scope; routing through GetOrCreateActorSystem (instead of a
        // plain singleton factory over .ActorSystem) means the first resolve CREATES the system
        // rather than caching a null if a probe wins the startup race.
        builder.Services.AddSingleton<Akka.Actor.ActorSystem>(sp =>
            sp.GetRequiredService<AkkaHostedService>().GetOrCreateActorSystem());

        // Simultaneous-cold-start split-brain guard (dark switch
        // ScadaBridge:Cluster:BootstrapGuard:Enabled, default off). Registered AFTER the
        // AkkaHostedService/ActorSystem bridge so its StartAsync resolves the system the main path
        // built; when the guard is off it no-ops (Akka auto-joined from config seeds), so registering
        // it unconditionally is safe. When on, the node started unjoined (empty HOCON seed list, see
        // AkkaHostedService.BuildHocon) and this coordinator issues the single reachability-gated
        // JoinSeedNodes. It resolves the ActorSystem lazily via GetOrCreateActorSystem so it never
        // races startup or caches a null.
        builder.Services.AddHostedService(sp => new ClusterBootstrapCoordinator(
            () => sp.GetRequiredService<AkkaHostedService>().GetOrCreateActorSystem(),
            sp.GetRequiredService<IOptions<ClusterOptions>>(),
            sp.GetRequiredService<IOptions<NodeOptions>>(),
            sp.GetRequiredService<ILogger<ClusterBootstrapCoordinator>>()));

        // Register the production active-node gate so standby-node gating is actually enforced
        // (the InboundApiEndpointFilter consults IActiveNodeGate and defaults to "allow" when none
        // is registered, which leaves the design's "central cluster only (active node)" guarantee
        // unenforced in deployed binaries). Since Gitea #80 this is the SHARED seam
        // (ZB.MOM.WW.Health.IActiveNodeGate) with the shared AkkaActiveNodeGate, unscoped like the
        // active-node check above — both route to ClusterActiveNode, so the inbound API and the
        // /health/active endpoint Traefik routes against can never disagree on which node is
        // active. Warm-up stays safe: the gate resolves the ActorSystem through the singleton
        // bridge above and answers false until this member is Up (and on any cluster-init fault).
        builder.Services.AddSingleton<ZB.MOM.WW.Health.IActiveNodeGate>(
            sp => new ZB.MOM.WW.Health.Akka.AkkaActiveNodeGate(sp));

        // Warm the inbound method compile cache in the BACKGROUND once the host is running and
        // this node is the active one (review 2026-10-21 05/U3). Registered after
        // AkkaHostedService so it never sits in front of the cluster join; the eager compile that
        // used to block here before app.RunAsync() is gone.
        builder.Services.AddHostedService<ZB.MOM.WW.ScadaBridge.InboundAPI.InboundMethodWarmupService>();

        // Admin-triggered manual failover of the central pair (Health page control,
        // decision 2026-07-22). Central-only: the seam is declared in CentralUI so that
        // project stays Akka-free, and only this branch has a cluster to act on.
        builder.Services.AddSingleton<
            ZB.MOM.WW.ScadaBridge.CentralUI.Services.IManualFailoverService, AkkaManualFailoverService>();

        // Cluster node status provider scoped to the Central role — feeds the
        // CentralHealthReportLoop so the central cluster appears on /monitoring/health.
        builder.Services.AddSingleton<IClusterNodeProvider>(sp =>
        {
            var akkaService = sp.GetRequiredService<AkkaHostedService>();
            return new AkkaClusterNodeProvider(akkaService, "Central");
        });

        // Options binding
        SiteServiceRegistration.BindSharedOptions(builder.Services, builder.Configuration);
        builder.Services.Configure<SecurityOptions>(builder.Configuration.GetSection("ScadaBridge:Security"));
        builder.Services.AddOptions<InboundApiOptions>()
            .Bind(builder.Configuration.GetSection("ScadaBridge:InboundApi"))
            .ValidateOnStart();
        builder.Services.TryAddEnumerable(
            ServiceDescriptor.Singleton<IValidateOptions<InboundApiOptions>, InboundApiOptionsValidator>());
        builder.Services.Configure<DeploymentManagerOptions>(
            builder.Configuration.GetSection(ZB.MOM.WW.ScadaBridge.DeploymentManager.ServiceCollectionExtensions.OptionsSection));

        var app = builder.Build();
        hostStopping = app.Lifetime.ApplicationStopping;

        // Apply or validate database migrations (skip when running in test harness)
        if (!string.Equals(configuration["ScadaBridge:Database:SkipMigrations"], "true", StringComparison.OrdinalIgnoreCase))
        {
            var isDevelopment = app.Environment.IsDevelopment()
                || string.Equals(Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"), "Development", StringComparison.OrdinalIgnoreCase);
            var migrationLogger = app.Services
                .GetRequiredService<ILoggerFactory>()
                .CreateLogger(typeof(MigrationHelper).FullName!);

            // Tolerate a database that is briefly unreachable at boot
            // (e.g. app and DB containers starting together) with a bounded
            // exponential backoff before failing fatally.
            // Only connection-class (transient) faults are retried — a
            // schema-version mismatch is permanent and must fail fast on attempt 1.
            // Thread the host's ApplicationStopping token into both the
            // migration call itself and the inter-attempt Task.Delay so a SIGTERM
            // during the bounded-retry window (~2 min worst-case) tears down
            // cleanly instead of being ignored until the loop exhausts.
            await StartupRetry.ExecuteWithRetryAsync(
                "database-migration",
                async ct =>
                {
                    using var scope = app.Services.CreateScope();
                    var dbContext = scope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>();
                    await MigrationHelper.ApplyOrValidateMigrationsAsync(dbContext, isDevelopment, migrationLogger, ct);
                },
                maxAttempts: 8,
                initialDelay: TimeSpan.FromSeconds(2),
                migrationLogger,
                isTransient: StartupRetry.IsTransientDatabaseFault,
                cancellationToken: app.Lifetime.ApplicationStopping);
        }

        // Middleware pipeline

        // Trusted-proxy forwarded headers (arch-review R2 N2): behind Traefik every client
        // shares the proxy's IP, collapsing the LoginThrottle's {username}|{ip} keys onto
        // one address (per-IP isolation gone + cheap username-lockout DoS). When enabled,
        // X-Forwarded-For from the CONFIGURED proxies only is honored. MUST run first —
        // everything downstream keys on Connection.RemoteIpAddress.
        var forwardedOptions = builder.Configuration
            .GetSection(ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSecurityOptions.SectionName)
            .Get<ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSecurityOptions>() ?? new();
        if (ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSetup.Build(forwardedOptions) is { } fho)
            app.UseForwardedHeaders(fho);

        app.UseWebSockets();
        app.UseRouting();
        app.UseCentralListenerIsolation(centralGrpcPort, centralMetricsPort, httpPorts);
        app.UseAuthentication();
        app.UseAuthorization();
        app.UseAntiforgery();

        // Emit one InboundRequest/InboundAuthFailure
        // audit row per call into the inbound API. Placed AFTER UseAuthentication/
        // UseAuthorization so any HttpContext.User the framework populates is in
        // place, and scoped to the /api/ prefix so it never observes the Central UI,
        // Management API, SignalR hubs, or health endpoints. The endpoint handler
        // is responsible for stashing the resolved API key name on
        // HttpContext.Items (see AuditWriteMiddleware.AuditActorItemKey) AFTER its
        // in-handler API key validation succeeds.
        // Scope the audit middleware to the inbound API method
        // route (/api/{methodName}) and explicitly exclude the management/audit
        // sub-trees that share the /api prefix. Without these exclusions the
        // middleware would emit a spurious ApiInbound audit row for every
        // /api/audit/query and /api/audit/export call (and would treat audit-log
        // reads as inbound script invocations — recursive write-on-read). The
        // POST-only filter rules out the GET routes on /api/audit, /api/centralui,
        // /api/script-analysis even if a future route is added under those
        // prefixes with the same verb; the explicit prefix excludes still belt-
        // and-brace POST-y additions there.
        app.UseWhen(
            ctx => ctx.Request.Path.StartsWithSegments("/api")
                && !ctx.Request.Path.StartsWithSegments("/api/audit")
                && !ctx.Request.Path.StartsWithSegments("/api/centralui")
                && !ctx.Request.Path.StartsWithSegments("/api/script-analysis")
                && !ctx.Request.Path.StartsWithSegments("/api/management")
                && HttpMethods.IsPost(ctx.Request.Method),
            branch => branch.UseAuditWriteMiddleware());

        // Map the canonical three-tier health endpoints in one call:
        //   /health/ready  — Ready-tagged checks (database + akka-cluster + required-singletons).
        //                    REQ-HOST-4a defines readiness as cluster membership + DB connectivity,
        //                    explicitly NOT active-node status, so the active-node check is excluded
        //                    (a fully operational standby central node still reports ready).
        //   /health/active — Active-tagged checks (active-node = oldest Up member + database
        //                    reachable); returns 200 only on the singleton-host node with a live
        //                    DB; used by Traefik for routing.
        //   /healthz       — bare process liveness; runs no checks (always 200 while the process
        //                    is up). New tier added by adopting the shared library.
        // All three are anonymous and use the canonical ZbHealthWriter JSON output.
        app.MapZbHealth();

        // Observability — mount the always-on Prometheus /metrics scrape endpoint.
        // AddZbTelemetry (in SiteServiceRegistration.BindSharedOptions) wires the OTel
        // Resource + standard instrumentation + Prometheus exporter; this exposes them.
        // Requires endpoint routing (app.UseRouting() above).
        app.MapZbMetrics();

        // Central-hosted gRPC control plane (T1A.2) — the site→central CentralControlService.
        // Runs on the dedicated h2c listener configured above (default :8083), gated by
        // CentralControlAuthInterceptor and readiness-gated by the service itself (Unavailable
        // until AkkaHostedService hands the CentralCommunicationActor over via SetReady). It
        // shares the app's endpoint routing with the HTTP/1 surface; Kestrel steers each
        // connection to the right pipeline by listener/protocol.
        app.MapGrpcService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.CentralControlGrpcService>();

        // The pull-only secrets hub that used to be mapped here is retired.
        // Sites receive secrets exclusively via Deploy Artifacts; there is no hub endpoint to map
        // on central and nothing left for a site to sweep. See SecretsRegistration.

        app.MapStaticAssets();
        app.MapCentralUI<ZB.MOM.WW.ScadaBridge.Host.Components.App>();
        app.MapInboundAPI();
        app.MapManagementAPI();
        // Audit Log: CLI-facing /api/audit/{query,export} routes. Same
        // Basic-Auth + LDAP mechanism as /management; gated on the OperationalAudit
        // / AuditExport role sets.
        app.MapAuditAPI();
        // Notify-and-fetch deploy: site-facing token-gated fetch of a staged
        // deployment's flattened config. Machine-to-machine — AllowAnonymous, gated
        // solely by the per-deployment X-Deployment-Token (no central FallbackPolicy).
        app.MapDeploymentConfigAPI();
        app.MapHub<ZB.MOM.WW.ScadaBridge.ManagementService.DebugStreamHub>("/hubs/debug-stream");

        // Inbound API method scripts are pre-compiled by InboundMethodWarmupService after startup
        // (background, active node only); the per-request lazy compile guarantees correctness.

        await app.RunAsync();
    }
    else if (nodeRole.Equals("Site", StringComparison.OrdinalIgnoreCase))
    {
        var builder = WebApplication.CreateBuilder(args);
        builder.Configuration.AddConfiguration(configuration);

        // Captive-dependency validation (deferred register row 50) — the SITE half of the
        // pair. This branch is the one that actually gains coverage: the docker rig long
        // set ASPNETCORE_ENVIRONMENT=Development on the central containers but NOT on the
        // site containers, so a site node built its provider unvalidated. Register row 63
        // closed that — all six rig site containers (and docker-env2's two) now set it, so
        // this validation actually runs there.
        builder.Host.UseDefaultServiceProvider(ServiceProviderValidation.Configure);

        // Serilog
        builder.Host.UseSerilog();

        // Windows Service support (no-op when not running as a Windows Service)
        builder.Host.UseWindowsService();

        // GrpcPort from the bound NodeOptions (default 8083)
        var grpcPort = nodeOptions.GrpcPort;

        // MetricsPort from the bound NodeOptions (default 8084).
        // Separate HTTP/1.1 listener so a standard HTTP/1.1 Prometheus scraper can
        // reach /metrics; the gRPC port stays HTTP/2-only below. The default is
        // 8084 — distinct from RemotingPort (8082, Akka) and GrpcPort (8083) so the
        // metrics listener never collides with the Akka remoting port on site nodes.
        var metricsPort = nodeOptions.MetricsPort;

        // Configure Kestrel for HTTP/2 only on the gRPC port (plus the h2 keepalive that
        // makes a half-open inbound stream detectable) — see SiteKestrelSetup.
        builder.WebHost.ConfigureKestrel(options =>
            SiteKestrelSetup.Configure(options, grpcPort, metricsPort));

        // gRPC server registration. Two interceptors, two disjoint service prefixes, two
        // separate keys — neither one's absence weakens the other:
        //
        //   LocalDbSyncAuthInterceptor  gates /localdb_sync.v1.LocalDbSync/  (LocalDb:Replication:ApiKey)
        //   ControlPlaneAuthInterceptor gates /sitestream.SiteStreamService/ (ScadaBridge:Communication:GrpcPsk)
        //
        // Both are fail-closed: an unset key closes that surface rather than opening it. For
        // LocalDb that means replication simply does not start; for the control plane it means
        // this node serves no streams and no audit pulls until a key is configured, which is
        // why every environment must carry one before running this build.
        builder.Services.AddGrpc(options =>
        {
            options.MaxReceiveMessageSize = ZB.MOM.WW.ScadaBridge.Communication.Grpc.GrpcMessageLimits.MaxMessageBytes;
            options.MaxSendMessageSize = ZB.MOM.WW.ScadaBridge.Communication.Grpc.GrpcMessageLimits.MaxMessageBytes;
            options.Interceptors.Add<LocalDbSyncAuthInterceptor>();
            options.Interceptors.Add<ControlPlaneAuthInterceptor>();
        });
        builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
        // Site command plane (central→site) — the gRPC peer of SiteStreamGrpcServer. Both share
        // the h2c listener and the ControlPlaneAuthInterceptor PSK gate; both are readiness-gated
        // (SetReady, below).
        builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteCommandGrpcService>();

        // Existing site service registrations (this is also where LocalDb and its
        // replication engine are registered — see SiteServiceRegistration)
        SiteServiceRegistration.Configure(builder.Services, builder.Configuration);

        var app = builder.Build();
        hostStopping = app.Lifetime.ApplicationStopping;

        // Endpoint routing middleware. The gRPC service mapping below and the
        // /metrics scrape endpoint both run on endpoint routing, so UseRouting()
        // must be present before the Map* calls on the site role.
        app.UseRouting();

        // Map the canonical three-tier health endpoints on the site node:
        //   /health/ready  — akka-cluster (site-pair membership, and the cluster-view data the
        //                    family overview dashboard reads) + localdb (the consolidated site
        //                    store — the only database a site node has).
        //   /health/active — active-node = this site pair's primary; 200 on the primary, 503 on
        //                    the standby, the same contract central publishes.
        //   /healthz       — bare process liveness.
        // Checks are registered in SiteServiceRegistration.Configure. All three endpoints are
        // anonymous and use the canonical ZbHealthWriter JSON; the site pipeline runs no
        // authentication middleware and has no FallbackPolicy, so nothing extra is needed to keep
        // them reachable. They are served on the HTTP/1.1 listener (metricsPort, default :8084)
        // alongside /metrics — the gRPC listener is HTTP/2-only and unusable by a plain HTTP client.
        app.MapZbHealth();

        // Observability — mount the always-on Prometheus /metrics scrape endpoint.
        // AddZbTelemetry (in SiteServiceRegistration.Configure → BindSharedOptions)
        // wires the OTel Resource + standard instrumentation + Prometheus exporter;
        // this exposes them on the site node too.
        app.MapZbMetrics();

        // Map gRPC service — resolves the singleton SiteStreamGrpcServer from DI
        app.MapGrpcService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();

        // Map the site command plane (central→site) alongside it, on the same gated listener.
        app.MapGrpcService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteCommandGrpcService>();

        // The passive half of LocalDb replication: the peer node dials THIS endpoint.
        // It shares the Kestrel h2c listener the site gRPC server already uses, so no
        // listener or port changes are needed. Mapping it is harmless with no peer
        // configured — nothing dials it, and LocalDbSyncAuthInterceptor (registered on
        // AddGrpc above) rejects anything that tries until an ApiKey is configured.
        app.MapZbLocalDbSync();

        // Site-shutdown ordering. ApplicationStopping
        // fires BEFORE IHostedService.StopAsync runs, so the gRPC server
        // refuses new streams (Unavailable) and cancels every active stream
        // here — clients observe a clean Cancelled and reconnect — and only
        // THEN does AkkaHostedService run CoordinatedShutdown and tear down
        // actors. Without this hand-off, in-flight streams go silent and only
        // time out via gRPC keepalive (~25 s), violating the documented
        // four-step sequence.
        var siteLifetime = app.Services.GetRequiredService<Microsoft.Extensions.Hosting.IHostApplicationLifetime>();
        var siteGrpcServer = app.Services.GetRequiredService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
        siteLifetime.ApplicationStopping.Register(() => siteGrpcServer.CancelAllStreams());

        await app.RunAsync();
    }
    else
    {
        throw new InvalidOperationException($"Unknown role: {nodeRole}. Must be 'Central' or 'Site'.");
    }
}
catch (Exception ex) when (Program.IsGracefulShutdown(ex, hostStopping))
{
    // Routine stop: shutdown was requested and the host unwound with a cancellation. Not a fatal exit.
    Log.Information("ScadaBridge host stopped (shutdown requested)");
}
catch (Exception ex)
{
    Log.Fatal(ex, "ScadaBridge host terminated unexpectedly");
    throw;
}
finally
{
    await Log.CloseAndFlushAsync();
}

/// <summary>
/// Exposes the auto-generated Program class for test infrastructure (e.g. WebApplicationFactory).
/// </summary>
public partial class Program
{
    /// <summary>
    /// True when <paramref name="ex"/> is the cancellation a routine host stop produces: an
    /// <see cref="OperationCanceledException"/> (incl. <see cref="TaskCanceledException"/>) escaping
    /// <c>RunAsync</c> while the host's <c>ApplicationStopping</c> token is already cancelled
    /// (ScadaBridge#76). Anything else - a non-cancellation exception, or a cancellation with no
    /// shutdown requested - is still a genuine fatal exit and must keep logging <c>[FTL]</c>.
    /// </summary>
    /// <param name="ex">The exception that escaped the host run.</param>
    /// <param name="hostStopping">The host's <c>ApplicationStopping</c> token (default if the host never built).</param>
    /// <returns>True when the exception is the expected cancellation of a requested shutdown.</returns>
    internal static bool IsGracefulShutdown(Exception ex, CancellationToken hostStopping) =>
        ex is OperationCanceledException && hostStopping.IsCancellationRequested;

    /// <summary>
    /// Extracts the distinct TCP ports from an ASP.NET Core server-URLs string (the
    /// <c>ASPNETCORE_URLS</c> / <c>--urls</c> value). Port-only view of
    /// <see cref="ParseHttpBindEndpoints"/>; kept for callers that only need the ports.
    /// </summary>
    /// <param name="serverUrls">The server-URLs string; may be null/empty.</param>
    /// <returns>The distinct ports, in first-seen order.</returns>
    internal static IReadOnlyList<int> ParseHttpBindPorts(string? serverUrls) =>
        ParseHttpBindEndpoints(serverUrls).Select(e => e.Port).ToList();

    /// <summary>
    /// Extracts the distinct (port, scheme) endpoints from an ASP.NET Core server-URLs string
    /// (the <c>ASPNETCORE_URLS</c> / <c>--urls</c> value, e.g. <c>"http://+:5000"</c> or a
    /// semicolon-separated list). Used to re-declare central's HTTP surface after the gRPC
    /// listener switches Kestrel into explicit-endpoints mode - see the call site's warning.
    /// </summary>
    /// <remarks>
    /// Bind hosts (<c>+</c>, <c>*</c>, <c>0.0.0.0</c>, <c>[::]</c>, a hostname) are irrelevant
    /// here because the caller re-binds via <c>ListenAnyIP</c>. The scheme IS kept (ScadaBridge#108):
    /// dropping it used to make <c>https://+:443</c> serve plaintext. A URL with no explicit port
    /// falls back to the scheme default (80/443). Unparseable entries are skipped rather than
    /// throwing. The same port listed with two schemes is a configuration error and throws.
    /// </remarks>
    /// <param name="serverUrls">The server-URLs string; may be null/empty.</param>
    /// <returns>The distinct endpoints, in first-seen order.</returns>
    internal static IReadOnlyList<HttpBindEndpoint> ParseHttpBindEndpoints(string? serverUrls)
    {
        var endpoints = new List<HttpBindEndpoint>();
        if (string.IsNullOrWhiteSpace(serverUrls))
        {
            return endpoints;
        }

        foreach (var raw in serverUrls.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
        {
            int port;
            var isHttps = raw.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
            // Uri can't parse the wildcard hosts Kestrel accepts (+, *), so normalize them
            // to a placeholder host before parsing; the host is discarded anyway.
            var normalized = raw.Replace("://+", "://placeholder").Replace("://*", "://placeholder");
            if (Uri.TryCreate(normalized, UriKind.Absolute, out var uri))
            {
                port = uri.Port; // Uri fills the scheme default (80/443) when none is given.
            }
            else
            {
                // Last-ditch: pull the port after the final ':' (handles odd inputs Uri rejects).
                var colon = raw.LastIndexOf(':');
                if (colon < 0 || !int.TryParse(raw.AsSpan(colon + 1), out port))
                {
                    continue;
                }
            }

            if (port is > 0 and <= 65535)
            {
                var existing = endpoints.FindIndex(e => e.Port == port);
                if (existing < 0)
                {
                    endpoints.Add(new HttpBindEndpoint(port, isHttps));
                }
                else if (endpoints[existing].IsHttps != isHttps)
                {
                    throw new InvalidOperationException(
                        $"ASPNETCORE_URLS lists port {port} with both http and https schemes; a port is one or the other.");
                }
            }
        }

        return endpoints;
    }
}