lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

using Akka.Actor;
using Akka.Hosting;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using Serilog;
using ZB.MOM.WW.LocalDb.Replication;
using Serilog.Events;
using ZB.MOM.WW.OtOpcUa.AdminUI;
using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
using ZB.MOM.WW.OtOpcUa.AdminUI.Components;
using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Services;
using ZB.MOM.WW.OtOpcUa.ControlPlane;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Host;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.Drivers;
using ZB.MOM.WW.OtOpcUa.Host.Grpc;
using ZB.MOM.WW.OtOpcUa.Host.Engines;
using ZB.MOM.WW.OtOpcUa.Host.Health;
using ZB.MOM.WW.OtOpcUa.Host.Logging;
using ZB.MOM.WW.OtOpcUa.Host.Observability;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Recorder;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
using ZB.MOM.WW.OtOpcUa.Runtime.Scripting;
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
using ZB.MOM.WW.OtOpcUa.Runtime;
using ZB.MOM.WW.Auth.Abstractions.Roles;
using ZB.MOM.WW.OtOpcUa.Security;
using ZB.MOM.WW.OtOpcUa.Security.Endpoints;
using ZB.MOM.WW.OtOpcUa.Security.Ldap;
using ZB.MOM.WW.Configuration;
using ZB.MOM.WW.Telemetry.Serilog;
using ZB.MOM.WW.OtOpcUa.AdminUI.Api;
using Microsoft.Extensions.DependencyInjection;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Configuration;
using ZB.MOM.WW.Secrets.DependencyInjection;
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
using ZB.MOM.WW.Secrets.Sqlite;

// Roles drive the entire conditional wiring below — see ZB.MOM.WW.OtOpcUa.Cluster.RoleParser.
var roles = RoleParser.Parse(Environment.GetEnvironmentVariable("OTOPCUA_ROLES"));
var hasAdmin = roles.Contains("admin");
var hasDriver = roles.Contains("driver");

var builder = WebApplication.CreateBuilder(args);

// Razor class library static assets (_content/<libname>/...) only auto-enable in
// the Development environment. Opt in explicitly so the AdminUI's CSS/JS works
// regardless of ASPNETCORE_ENVIRONMENT.
builder.WebHost.UseStaticWebAssets();

// Windows Service support: when the EXE is started by Service Control Manager (sc.exe),
// the host needs to call SetServiceStatus to keep the SCM happy. UseWindowsService()
// installs the WindowsServiceLifetime IFF WindowsServiceHelpers.IsWindowsService() is
// true at runtime — so it's safely a no-op when running as a console app or on Linux.
builder.Host.UseWindowsService(options => options.ServiceName = "OtOpcUaHost");

// Per-role appsettings overlay: appsettings.{role}.json (single role) or appsettings.admin-driver.json
// (both). Optional — base appsettings.json carries enough to boot if these don't exist.
var roleSuffix = roles.Length == 0 ? null : string.Join('-', roles.OrderBy(r => r, StringComparer.Ordinal));
if (roleSuffix is not null)
{
    builder.Configuration.AddJsonFile($"appsettings.{roleSuffix}.json", optional: true, reloadOnChange: true);

    // The overlay above is appended AFTER WebApplicationBuilder's default sources, so without this it
    // would outrank environment variables and command-line args — i.e. a baked role file could not be
    // overridden by a deployment env var (e.g. the dev compose's Security__Ldap__DevStubMode=true was
    // silently beaten by appsettings.admin.json's DevStubMode:false). Re-append env vars then args last
    // so deployment overrides keep top precedence; the overlay still wins over base appsettings.json.
    builder.Configuration.AddEnvironmentVariables();
    builder.Configuration.AddCommandLine(args);
}

// Pre-host ${secret:} expansion: rewrites ${secret:name} tokens in the assembled configuration
// BEFORE the host is built, so every downstream binder/validator sees resolved plaintext.
// Placement matters: this runs AFTER all config providers are assembled (base appsettings, the
// role overlay, and the env-vars + command-line re-append above) but BEFORE anything READS a
// config value (AddZbSerilog's ReadFrom.Configuration, AddOtOpcUaConfigDb, and the first
// ValidateOnStart bind all follow). With no ${secret:} tokens present this is a no-op pass;
// it also migrates the secrets SQLite store on startup.
// ASP0000: DELIBERATE throwaway container — expansion must run before builder.Build(), so it
// cannot use the app's DI. Fully disposed here; shares no singletons with the host container.
#pragma warning disable ASP0000
await using (var secretsProvider = new ServiceCollection()
    .AddZbSecrets(builder.Configuration, "Secrets")
    .BuildServiceProvider())
#pragma warning restore ASP0000
{
    await secretsProvider.GetRequiredService<SqliteSecretsStoreMigrator>().MigrateAsync(default);
    var resolver = secretsProvider.GetRequiredService<ISecretResolver>();
    await new SecretReferenceExpander(resolver)
        .ExpandConfigurationAsync((IConfigurationRoot)builder.Configuration, default);
}

// Anchor the process working directory to the install directory (AppContext.BaseDirectory) so every
// relative runtime path resolves under the install dir rather than the service's startup CWD. A Windows
// service starts with CWD=C:\Windows\System32, which otherwise scattered the Serilog rolling-file sink
// (appsettings "logs/otopcua-.log") into System32 and made on-box diagnosis awkward; the same anchor also
// roots the script log, the historization outbox, and any other relative artifact path. Done AFTER
// WebApplication.CreateBuilder (which has already resolved the content root + appsettings.json) and BEFORE
// AddZbSerilog (which instantiates the file sink), so only path resolution moves — config loading is
// untouched. UseWindowsService already roots the content root at the base dir, so this only affects sinks.
Directory.SetCurrentDirectory(AppContext.BaseDirectory);

// Serilog — shared ZB.MOM.WW.Telemetry bootstrap. Sinks (Console + rolling daily file)
// now live in appsettings.json (ReadFrom.Configuration); AddZbSerilog layers in the
// shared NodeHostname / TraceContext / Redaction enrichers and trace correlation.
builder.AddZbSerilog(o => o.ServiceName = "otopcua");

// Windows-service registration is handled at install time by scripts/install/Install-Services.ps1
// rather than in-process, so the binary stays cross-platform-compilable.

// ConfigDb is admin-only now (per-cluster mesh Phase 4). Only the admin role owns a ConfigDb
// connection string; a driver-only node holds none at all and gets its deployed configuration from
// central over the Phase-3 fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it
// unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string.
// A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected.
// #520 — bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it
// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a
// startup error.
builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
// #520 stage 5 — Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config
// change rather than a redeploy, and what it swaps rather than adds.
builder.Services.Configure<AclOptions>(builder.Configuration.GetSection(AclOptions.SectionName));

if (hasAdmin)
    builder.Services.AddOtOpcUaConfigDb(builder.Configuration);
else
    Log.Information("Driver-only node — no ConfigDb registered; configuration via ConfigSource:Mode=FetchAndCache");

builder.Services.AddOtOpcUaCluster(builder.Configuration);

// Validate LdapOptions unconditionally so ANY role node (admin-only, driver-only, or fused)
// fails fast at startup on a misconfigured or insecure-transport LDAP section.
// The validator's own guard (if !Enabled || DevStubMode) keeps it a safe no-op on disabled/dev
// configs. Previously this was inside the hasDriver block, which left admin-only nodes without
// the startup check (AddOtOpcUaAuth binds LdapOptions but does not attach a validator).
builder.Services.AddValidatedOptions<LdapOptions, LdapOptionsValidator>(builder.Configuration, LdapOptions.SectionName);

if (hasDriver)
{
    builder.Services.AddOtOpcUaRuntime();

    // Config-gated durable alarm-historian sink. When the AlarmHistorian section is enabled this
    // overrides the NullAlarmHistorianSink default from AddOtOpcUaRuntime (last registration wins)
    // with a LocalDbStoreAndForwardSink draining to the gateway SendEvent writer. The alarm-write path
    // targets the SAME single gateway as the read path, so its connection (endpoint/key/TLS) is sourced
    // from the ServerHistorian section. AlarmHistorianOptions supplies only the Enabled gate + the
    // SQLite store-and-forward knobs (consumed inside AddAlarmHistorian) — it carries no connection fields.
    // Runtime owns the gating + Sqlite construction; the Host supplies the concrete gateway downstream
    // via the driver factory (which owns the package-client adapter). The writer builds its OWN gateway
    // channel — a second channel to the same sidecar: sharing one channel with the read path would force
    // the read-side GatewayHistorianDataSource to stop owning + disposing its client (regressing the read
    // cutover), and a second channel to a co-located sidecar is cheap (the gateway pools the historian
    // sessions server-side).
    // Fail-fast startup validation for the ServerHistorian section (archreview 06/S-11). A
    // provably-crashing config — an enabled historian (or an enabled AlarmHistorian, which sources its
    // gateway connection from THIS section) with an empty/malformed Endpoint — now fails host start with
    // a named OptionsValidationException instead of a raw UriFormatException crash-loop out of the gateway
    // client factory. Driver-block-only (mirrors OpcUaApplicationHostOptionsValidator below): admin-only
    // nodes never build the historian singletons, so they must not be failed by a section they never
    // consume. ApiKey/MaxTieClusterOverfetch stay warnings (they degrade, not crash) — see Validate().
    builder.Services.AddValidatedOptions<ServerHistorianOptions, ServerHistorianOptionsValidator>(
        builder.Configuration, ServerHistorianOptions.SectionName);

    var serverHistorianOptions = builder.Configuration
        .GetSection(ServerHistorianOptions.SectionName).Get<ServerHistorianOptions>()
        ?? new ServerHistorianOptions();
    foreach (var warning in serverHistorianOptions.Validate())
        Log.Warning("ServerHistorian misconfiguration detected at startup: {Warning}", warning);

    // Residual "unusual but valid" corner (archreview 06/S-11): an alarm-history-only deployment
    // (AlarmHistorian:Enabled=true while ServerHistorian:Enabled=false) legitimately sources its gateway
    // connection from the disabled ServerHistorian section. That is allowed (the endpoint was already
    // validated above), but log it so the mode is not a silent surprise: HistoryReads stay GoodNoData,
    // and a wrong endpoint/key surfaces only as SQLite store-and-forward retry/dead-letter growth.
    if (builder.Configuration.GetValue<bool>("AlarmHistorian:Enabled") && !serverHistorianOptions.Enabled)
        Log.Warning(
            "AlarmHistorian is enabled but ServerHistorian is disabled — the alarm-history sink is sourcing its " +
            "gateway connection (endpoint/key/TLS) from the disabled ServerHistorian section; HistoryReads stay GoodNoData.");

    builder.Services.AddAlarmHistorian(
        builder.Configuration,
        (_, sp) => GatewayHistorian.CreateAlarmWriter(serverHistorianOptions, sp));

    // Node-local LocalDb: caches the deployed-configuration artifact so this node can boot from
    // its last-known-good config when central SQL is unreachable, and optionally replicates that
    // cache to its redundant pair peer. Driver-role only — admin-only nodes have nothing to cache,
    // and registering here would impose LocalDb:Path-required-or-no-boot on them for nothing.
    // Storage ships unconditionally; replication stays inert until LocalDb:Replication:PeerAddress
    // (initiator) / LocalDb:SyncListenPort (listener) are set. See LocalDbRegistration.
    builder.Services.AddOtOpcUaLocalDb(builder.Configuration);

    // Node-side gRPC artifact fetcher (per-cluster mesh Phase 3). Resolved + used by DriverHostActor
    // only under ConfigSource:Mode = FetchAndCache; idle (never invoked) in the default Direct mode, so
    // registering it unconditionally on driver nodes is harmless. IDisposable — DI disposes its cached
    // h2c channels at shutdown.
    builder.Services.AddSingleton<IDeploymentArtifactFetcher, GrpcDeploymentArtifactFetcher>();

    // gRPC server plumbing (AddGrpc + the two path-scoped auth interceptors) is registered once,
    // below, for hasDriver || hasAdmin — the LocalDb passive sync endpoint (driver) and the
    // ConfigServe artifact endpoint (admin) share one pipeline.

    // Config-gated server-side HistoryRead backend. When the ServerHistorian section is enabled this
    // overrides the NullHistorianDataSource default from AddOtOpcUaRuntime (last registration wins) with
    // a read-only HistorianGateway-backed data source the node manager's HistoryRead overrides
    // block-bridge to. The factory lives in the Gateway driver (which owns the package-client adapter
    // and the ServerHistorianOptions -> client-options mapping); Runtime owns the gating.
    builder.Services.AddServerHistorian(
        builder.Configuration,
        (opts, sp) => GatewayHistorian.CreateDataSource(opts, sp));

    // Config-gated historian tag provisioning. When the ServerHistorian section is enabled this registers the
    // gateway-backed GatewayTagProvisioner (overriding the NullHistorianProvisioning default) so that deploying
    // a historized tag auto-ensures it in the historian via the gateway's EnsureTags. The AddressSpaceApplier
    // (constructed in WithOtOpcUaRuntimeActors) resolves IHistorianProvisioning and fires a non-blocking
    // EnsureTagsAsync for added historized tags on every deploy; without this registration the applier falls
    // back to the no-op NullHistorianProvisioning and provisioning never reaches the gateway. Gated on the SAME
    // ServerHistorian:Enabled flag as the read path — both target the same single gateway.
    builder.Services.AddHistorianProvisioning(
        builder.Configuration,
        (opts, sp) => GatewayHistorian.CreateProvisioner(opts, sp));

    // Continuous historization of driver (non-Galaxy) tag values. Gated on ContinuousHistorization:Enabled
    // AND the ServerHistorian gateway being configured: the recorder drains driver-tag live values to the
    // SAME single gateway's WriteLiveValues SQL path, sourcing endpoint/key/TLS from the ServerHistorian
    // section (this section carries only the recorder + outbox knobs). When both are on, register the durable
    // crash-safe outbox + the gateway-backed live-value writer here; WithOtOpcUaRuntimeActors (below) spawns
    // the recorder actor itself, gated on the same options.
    var continuousHistorizationOptions = builder.Configuration
        .GetSection(ContinuousHistorizationOptions.SectionName).Get<ContinuousHistorizationOptions>()
        ?? new ContinuousHistorizationOptions();
    foreach (var warning in continuousHistorizationOptions.Validate())
        Log.Warning("ContinuousHistorization misconfiguration detected at startup: {Warning}", warning);
    if (serverHistorianOptions.Enabled && continuousHistorizationOptions.Enabled)
    {
        // Register the bound options so WithOtOpcUaRuntimeActors can gate the recorder spawn on Enabled.
        builder.Services.AddSingleton(continuousHistorizationOptions);

        // Durable, crash-safe FasterLog outbox (the historization crash boundary). Built via the factory so
        // the container OWNS disposal (FasterLogHistorizationOutbox is IDisposable). Binding the observable
        // outbox depth/dropped gauges here (once, on first resolution) keeps the live instance behind them.
        builder.Services.AddSingleton<IHistorizationOutbox>(_ =>
        {
            var commitMode = Enum.TryParse<HistorizationCommitMode>(
                continuousHistorizationOptions.CommitMode, ignoreCase: true, out var parsedMode)
                ? parsedMode
                : HistorizationCommitMode.PerEntry;
            var outbox = new FasterLogHistorizationOutbox(
                continuousHistorizationOptions.OutboxPath,
                commitMode,
                continuousHistorizationOptions.CommitIntervalMs,
                continuousHistorizationOptions.Capacity);
            ContinuousHistorizationMetrics.BindOutbox(outbox);
            return outbox;
        });

        // Gateway-backed live-value writer over its OWN gRPC channel to the same single gateway (a second
        // channel to a co-located sidecar is cheap — the gateway pools the historian sessions server-side).
        builder.Services.AddSingleton<IHistorianValueWriter>(sp =>
            new GatewayHistorianValueWriter(
                HistorianGatewayClientAdapter.Create(
                    serverHistorianOptions, sp.GetRequiredService<ILoggerFactory>()),
                sp.GetRequiredService<ILogger<GatewayHistorianValueWriter>>()));
    }

    // Bind every cross-platform driver factory before AddAkka resolves IDriverFactory — replaces
    // the F7-default NullDriverFactory with a real DriverFactoryRegistryAdapter so DriverHostActor
    // can materialise real IDriver instances on deploy.
    builder.Services.AddOtOpcUaDriverFactories();

    // Deferred sink so Akka actors can resolve IOpcUaAddressSpaceSink at construction time —
    // the OPC UA hosted service swaps in a real SdkAddressSpaceSink once StandardServer has
    // started. Until then writes route through NullOpcUaAddressSpaceSink.
    builder.Services.AddSingleton<DeferredAddressSpaceSink>();
    builder.Services.AddSingleton<IOpcUaAddressSpaceSink>(sp =>
        sp.GetRequiredService<DeferredAddressSpaceSink>());

    // Same late-binding pattern for the ServiceLevel publisher — actor wants it at ctor time,
    // production SdkServiceLevelPublisher needs IServerInternal which only exists after Start.
    builder.Services.AddSingleton<DeferredServiceLevelPublisher>();
    builder.Services.AddSingleton<IServiceLevelPublisher>(sp =>
        sp.GetRequiredService<DeferredServiceLevelPublisher>());

    // F13c — bind UserName tokens to the same LDAP backend the Admin cookie/JWT flows use.
    // ILdapAuthService is registered by AddOtOpcUaAuth on admin nodes; on driver-only nodes
    // it isn't, so we register the LDAP options + service unconditionally for driver hosts
    // to keep parity. The LdapAdapter falls back to Deny on any backend error.
    // F8b — production virtual-tag evaluator (Roslyn-compiled scripts cached per expression).
    // Replaces the F8-default NullVirtualTagEvaluator so VirtualTagActor evaluates real user
    // scripts at runtime.
    builder.Services.AddSingleton<RoslynVirtualTagEvaluator>(sp =>
        new RoslynVirtualTagEvaluator(
            sp.GetRequiredService<ILoggerFactory>().CreateLogger<RoslynVirtualTagEvaluator>(),
            sp.GetRequiredService<ScriptRootLogger>()));
    builder.Services.AddSingleton<IVirtualTagEvaluator>(sp => sp.GetRequiredService<RoslynVirtualTagEvaluator>());

    // Script-log fan-out (Layer 0). The DPS publisher resolves the ActorSystem lazily so it never
    // races Akka startup. ScriptRootLogger wraps the composed pipeline (rolling scripts-*.log +
    // error mirror to the main log + script-logs DPS topic) for unambiguous DI resolution; it
    // injects into the Roslyn evaluators above.
    var scriptLogFilePath = builder.Configuration["Scripting:LogFilePath"] ?? "logs/scripts-.log";
    var scriptLogTopicMinLevel = Enum.TryParse<LogEventLevel>(
        builder.Configuration["Scripting:LogTopicMinLevel"], ignoreCase: true, out var parsedLevel)
        ? parsedLevel
        : LogEventLevel.Information;
    builder.Services.AddSingleton<IScriptLogPublisher>(sp =>
        new DpsScriptLogPublisher(
            () => sp.GetRequiredService<ActorSystem>(),
            sp.GetRequiredService<ITelemetryLocalHub>()));
    builder.Services.AddSingleton(sp => new ScriptRootLogger(
        ScriptRootLoggerFactory.Build(
            sp.GetRequiredService<IScriptLogPublisher>(), scriptLogFilePath, scriptLogTopicMinLevel, Serilog.Log.Logger)));

    // TryAdd so a fused admin+driver node (where AddOtOpcUaAuth also registers these) ends up
    // with exactly one descriptor; on a driver-only node these are the sole registrations.
    // OtOpcUaLdapAuthService is the app ILdapAuthService (Enabled switch + DevStubMode over the
    // shared ZB.MOM.WW.Auth.Ldap service). The data-plane authenticator resolves IGroupRoleMapper
    // <string> per call to turn the directory's groups into roles, so register it here for driver-
    // only nodes (AddOtOpcUaAuth registers it on admin nodes). OtOpcUaGroupRoleMapper also depends
    // on ILdapGroupRoleMappingService (the control-plane DB-grants CRUD surface) — Phase 4 gated
    // AddOtOpcUaConfigDb on hasAdmin, so a driver-only node has no ConfigDb to read it from. Bind
    // NullLdapGroupRoleMappingService (always "no DB grants") here via TryAdd so the mapper falls
    // back to the appsettings Security:Ldap:GroupToRole baseline; on a fused admin+driver node the
    // hasAdmin block above already ran AddOtOpcUaConfigDb's non-Try AddScoped<ILdapGroupRoleMapping
    // Service, LdapGroupRoleMappingService>, so this TryAdd is a no-op there and the real EF service
    // wins.
    // Note: AddValidatedOptions<LdapOptions, LdapOptionsValidator> is now registered unconditionally
    // above both role blocks so admin-only nodes also get fail-fast LDAP startup validation.
    builder.Services.TryAddSingleton<ILdapAuthService, OtOpcUaLdapAuthService>();
    builder.Services.TryAddScoped<ILdapGroupRoleMappingService, NullLdapGroupRoleMappingService>();
    builder.Services.TryAddScoped<IGroupRoleMapper<string>, OtOpcUaGroupRoleMapper>();

    // #520 stage 2 — the OPC UA DATA PLANE is machine-to-machine, so its identity is an API key
    // (static username "apikey", the otua_ credential as the password), not an LDAP user. The LDAP
    // registrations above are deliberately retained: they back ADMINUI LOGIN, which keeps LDAP.
    // Only IOpcUaUserAuthenticator moves.
    // #520 stage 4 — the node's data-plane authorization inputs (keys + ACL grants + cluster + auth
    // generation), refreshed by OpcUaPublishActor from every applied artifact. Registered on any
    // driver-role node because that is what serves an OPC UA data plane; the two interfaces resolve to
    // the SAME instance, since the writer and the readers must see one snapshot.
    if (hasDriver)
    {
        builder.Services.AddSingleton<DataPlaneAuthorityState>();
        builder.Services.AddSingleton<IDataPlaneAuthoritySource>(sp => sp.GetRequiredService<DataPlaneAuthorityState>());
        builder.Services.AddSingleton<IDataPlaneAuthoritySink>(sp => sp.GetRequiredService<DataPlaneAuthorityState>());

        // #520 stage 5 — the tries the ACL gate evaluates against are taken FROM the authority state, not
        // registered independently. The state builds the trie inside the same Apply() call that installs
        // the snapshot, so resolving any other cache here would hand the evaluator a permanently empty
        // one — and an empty trie denies everything, which looks exactly like "the ACLs are working" until
        // someone checks that an ALLOW is possible at all. Deriving it from the owner makes that mistake
        // unrepresentable.
        builder.Services.AddSingleton(sp => sp.GetRequiredService<DataPlaneAuthorityState>().Tries);
    }
    else
    {
        // The SOURCE half must always resolve — OtOpcUaServerHostedService takes it as a required ctor
        // parameter, and the built-in DI container does not fall back to a default value for a parameter
        // whose service is unregistered; it throws and the node fails to start. Same trap, and the same
        // fix, as the NullOpcUaUserAuthenticator registration below. Only the SINK is genuinely optional
        // (nothing publishes artifacts on a node with no driver role).
        builder.Services.AddSingleton<IDataPlaneAuthoritySource>(EmptyDataPlaneAuthoritySource.Instance);
        // Nothing installs a trie on a node with no driver role; an empty cache denies every scoped
        // decision, which is correct for a node that serves no data plane.
        builder.Services.AddSingleton<PermissionTrieCache>();
    }

    // #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because
    // OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves
    // constructors by whether every parameter is registered, not by whether the parameter type is nullable.
    // That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations above
    // both carry a comment about; an absent registration is a node that will not start.
    builder.Services.AddSingleton<IPermissionEvaluator>(sp =>
        new TriePermissionEvaluator(sp.GetRequiredService<PermissionTrieCache>()));
    builder.Services.AddSingleton(sp => new DataPlaneAuthorizer(
        sp.GetRequiredService<IDataPlaneAuthoritySource>(),
        sp.GetRequiredService<IPermissionEvaluator>(),
        sp.GetRequiredService<IOptions<AclOptions>>().Value.Enabled,
        sp.GetRequiredService<ILoggerFactory>().CreateLogger<DataPlaneAuthorizer>()));

    if (hasAdmin)
    {
        // A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect
        // on the very next request rather than on the next deployment. Preferred wherever it is available,
        // including on a fused admin+driver node.
        // The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds
        // the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which
        // ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration),
        // NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services
        // rule is what keeps the redundancy-singleton registration below from recursing into ActorSystem
        // construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter (legacy behaviour).
        builder.Services.AddSingleton<IApiKeyStore>(sp => new ConfigDbApiKeyStore(
            sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
            ConfigDbApiKeyStore.ResolveClusterId(sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)));
        builder.Services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();

        // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
        builder.Services.AddSingleton<IApiKeyAdminService>(sp => new ApiKeyAdminService(
            sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
            sp.GetRequiredService<IOptions<ApiKeyOptions>>().Value.Pepper));
    }
    else if (hasDriver)
    {
        // Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact.
        // Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName
        // token — safe, but it meant no machine client could authenticate against a site node at all.
        builder.Services.AddSingleton<IApiKeyStore, ArtifactApiKeyStore>();
        builder.Services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
    }
    else
    {
        // Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent —
        // OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so
        // an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start.
        builder.Services.AddSingleton<IOpcUaUserAuthenticator>(NullOpcUaUserAuthenticator.Instance);
    }

    // Bind + validate the OPC UA host options the same way (fail-fast at start via ValidateOnStart)
    // and let OtOpcUaServerHostedService consume the validated IOptions instance rather than
    // re-binding the section imperatively. Defaults pass; this guards explicit prod/env overrides.
    builder.Services.AddValidatedOptions<OpcUaApplicationHostOptions, OpcUaApplicationHostOptionsValidator>(
        builder.Configuration, "OpcUa");

    // Lazy ActorSystem accessor so OtOpcUaServerHostedService can resolve the DistributedPubSub
    // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
    // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
    builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
    builder.Services.AddHostedService<OtOpcUaServerHostedService>();
}

// Akka cluster bootstrap. Role-specific singletons are registered on the AkkaConfigurationBuilder
// from inside the configurator lambda. AddAkka spins the ActorSystem at host start.
builder.Services.AddAkka("otopcua", (ab, sp) =>
{
    ab.WithOtOpcUaClusterBootstrap(sp);
    // Secret-replication wire protocol → its own serializer, merged only when replication is on so a
    // non-replicating node carries no serializer bindings for messages it will never see. Appended
    // (Akka.Hosting's fallback merge — the same mode WithOtOpcUaClusterBootstrap uses for the base
    // config) rather than a raw Config.WithFallback, which would fight the builder's own assembly.
    // Without this the protocol DTOs would round-trip on Akka's default JSON serializer, leaving the
    // serializer that carries secret ciphertext an inherited default rather than an explicit choice.
    if (SecretsRegistration.IsReplicationEnabled(builder.Configuration))
        ab.AddHocon(AkkaSecretsReplication.SerializationConfig, HoconAddMode.Append);
    if (hasAdmin)
    {
        ab.WithOtOpcUaControlPlaneSingletons();
        ab.WithOtOpcUaSignalRBridges();
    }
    if (hasDriver)
    {
        ab.WithOtOpcUaRuntimeActors();
        // Per-cluster mesh Phase 6: re-home the redundancy-state singleton off the admin node onto
        // EVERY driver node, so each mesh elects its own pair-local Primary (a driver-only site pair
        // has no admin node to do it). It scopes to this node's own cluster-{ClusterId} role, falling
        // back to the driver role for legacy single-mesh / not-yet-migrated nodes (already pair-local
        // on a split mesh, since the mesh IS the pair). Runs on the fused central node too (it is
        // hasDriver), where it scopes to cluster-MAIN — exactly one redundancy singleton per node,
        // since the admin branch above no longer registers one. Pass AkkaClusterOptions (pure config),
        // NOT IClusterRoleInfo: this lambda runs WHILE the ActorSystem is being built, and
        // IClusterRoleInfo depends on the ActorSystem — resolving it here recurses into ActorSystem
        // construction and stack-overflows the host at boot. The singleton scope only needs the node's
        // configured cluster role, which AkkaClusterOptions carries.
        ab.WithOtOpcUaClusterRedundancySingleton(sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value);
    }
});

// Simultaneous-cold-start split-brain guard (dark switch Cluster:BootstrapGuard:Enabled, default off).
// Registered AFTER AddAkka so its StartAsync runs once Akka's hosted service has built (and, when the
// guard is off, already auto-joined) the ActorSystem. When on, the node started with no config seeds
// (BuildClusterOptions) and this coordinator issues the single reachability-gated JoinSeedNodes. It
// no-ops when the guard is off, so registering it unconditionally is safe.
builder.Services.AddHostedService(sp => new ClusterBootstrapCoordinator(
    () => sp.GetRequiredService<ActorSystem>(),
    sp.GetRequiredService<IOptions<AkkaClusterOptions>>(),
    sp.GetRequiredService<ILogger<ClusterBootstrapCoordinator>>()));

// Down-if-alone recovery watchdog (#459). Registered AFTER AddAkka so it starts after Akka's own
// hosted service has built the ActorSystem; it resolves the system lazily (never at construction) so
// it can't race startup. On an unexpected SBR self-down it stops the host so the service supervisor
// (sc.exe failure / docker restart: unless-stopped) restarts this node — the 2-node keep-oldest
// exit-and-rejoin recovery path.
builder.Services.AddHostedService(sp => new ActorSystemTerminationWatchdog(
    () => sp.GetRequiredService<ActorSystem>(),
    sp.GetRequiredService<IHostApplicationLifetime>(),
    sp.GetRequiredService<ILogger<ActorSystemTerminationWatchdog>>()));

if (hasAdmin)
{
    // Auth + AdminUI surface only mounted on admin-role nodes. Driver-only nodes have no UI.
    builder.Services.AddOtOpcUaAuth(builder.Configuration);
    builder.Services.AddAdminUI();
    // Test Connect probes back the AdminUI driver pages. The admin-operations singleton (role-pinned
    // to admin) resolves IEnumerable<IDriverProbe>, so admin-only nodes — which skip the hasDriver
    // block above — must wire the probe set here too, or every Test Connect returns "No probe
    // registered". Idempotent on fused admin,driver nodes (TryAddEnumerable de-dups).
    builder.Services.AddOtOpcUaDriverProbes();
    // Flow AuthenticationState through cascading parameters so <AuthorizeView/> works
    // in the static MainLayout footer and other components (e.g. Account.razor, Routes.razor).
    builder.Services.AddCascadingAuthenticationState();
    builder.Services.AddSignalR();
    builder.Services.AddOtOpcUaAdminClients();
    // Manual failover backs the Trigger-failover control on the cluster redundancy page. Admin-only:
    // it is driven from the UI, and the node it acts on is chosen from live cluster state, so any
    // admin node can drive it regardless of whether that node is the Primary. The ActorSystem comes
    // through the Func<> accessor below (never resolved at construction — this runs before Akka's
    // hosted service has built the system).
    builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
    builder.Services.AddSingleton<IManualFailoverService, ManualFailoverService>();
}

// Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI.
// Cluster replication is opt-in behind Secrets:Replication:Enabled (default false) — see SecretsRegistration.
builder.Services.AddOtOpcUaSecrets(builder.Configuration);

// hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
// probing it would throw when the readiness endpoint resolves the factory.
builder.Services.AddOtOpcUaHealth(hasAdmin);
builder.Services.AddOtOpcUaObservability(builder.Configuration);

// gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,
// mapped below on hasDriver && syncListenPort > 0), the ConfigServe artifact endpoint (admin-role,
// mapped on hasAdmin && configServeGrpcPort > 0), and the telemetry-stream endpoint (driver-role,
// mapped on hasDriver && telemetryListenPort > 0, per-cluster mesh Phase 5). Each interceptor is
// scoped strictly to its own service path, so all three can share one pipeline and each is a
// harmless pass-through when its service is unmapped. All three fail-closed with no ApiKey
// configured (LocalDbSyncAuthInterceptor, ConfigServeAuthInterceptor, TelemetryStreamAuthInterceptor)
// — the interceptor is the ONLY inbound auth on any endpoint (the libraries verify nothing).
// Registered whenever either role is present so an admin-only node still serves config and a
// driver-only node still gates sync + telemetry.
if (hasDriver || hasAdmin)
{
    builder.Services.AddGrpc(o =>
    {
        if (hasDriver)
        {
            o.Interceptors.Add<LocalDbSyncAuthInterceptor>();
            o.Interceptors.Add<TelemetryStreamAuthInterceptor>();
        }
        if (hasAdmin)
            o.Interceptors.Add<ConfigServeAuthInterceptor>();
    });
}

// ---------------------------------------------------------------------------------------------
// Dedicated h2c listeners (all default-OFF): the LocalDb sync endpoint (driver), the ConfigServe
// artifact endpoint (admin, per-cluster mesh Phase 3), and the telemetry-stream endpoint (driver,
// per-cluster mesh Phase 5).
//
// DANGER: any explicit Kestrel Listen* call makes Kestrel IGNORE ASPNETCORE_URLS/urls ENTIRELY
// (it logs "Overriding address(es)"). The host has no ConfigureKestrel today and binds solely via
// that configuration, so adding a listener naively would silently unbind the AdminUI + deploy API
// behind Traefik. Everything the host was already asked to serve is therefore re-bound explicitly
// in the same block — and, critically, EXACTLY ONCE: a fused admin+driver node can have ALL THREE
// dedicated ports set, and re-applying the existing surface per-port would double-bind it and throw
// "address already in use". So the existing surface is computed and applied once, then whichever of
// the three dedicated ports are configured are added on top.
//
// Each listener is HTTP/2-ONLY on purpose: every client speaks prior-knowledge h2c, which a cleartext
// Http1AndHttp2 endpoint cannot negotiate (there is no ALPN without TLS). Hence dedicated ports
// rather than multiplexing onto the main one.
//
// When all three ports are 0 (the default) none of this runs and URL binding is untouched.
// ---------------------------------------------------------------------------------------------
var syncListenPort = hasDriver ? LocalDbRegistration.SyncListenPort(builder.Configuration) : 0;
var configServeGrpcPort = hasAdmin ? builder.Configuration.GetValue<int>("ConfigServe:GrpcListenPort") : 0;
var telemetryListenPort = hasDriver ? builder.Configuration.GetValue<int>("Telemetry:GrpcListenPort") : 0;
if (syncListenPort > 0 || configServeGrpcPort > 0 || telemetryListenPort > 0)
{
    // Mirror Kestrel's own source precedence: URLS wins; else the HTTP_PORTS/HTTPS_PORTS bare-port
    // vars; else Kestrel's localhost:5000 default. Missing the HTTP_PORTS leg is not academic — the
    // aspnet:8.0+ base images set ASPNETCORE_HTTP_PORTS=8080 as the CONTAINER default in place of
    // ASPNETCORE_URLS, so a driver node that never sets URLS is really serving :8080 on all
    // interfaces. Falling straight through to localhost:5000 would silently move its health/metrics
    // surface to loopback (found on the docker-dev live gate).
    var configuredUrls = builder.Configuration["urls"]
                      ?? builder.Configuration["ASPNETCORE_URLS"];
    var existingBindings = KestrelHttpBinding.Parse(configuredUrls);

    if (existingBindings.Count == 0)
    {
        var httpPorts = builder.Configuration["ASPNETCORE_HTTP_PORTS"] ?? builder.Configuration["HTTP_PORTS"];
        var httpsPorts = builder.Configuration["ASPNETCORE_HTTPS_PORTS"] ?? builder.Configuration["HTTPS_PORTS"];
        existingBindings =
        [
            .. KestrelHttpBinding.ParsePorts(httpPorts, isSecure: false),
            .. KestrelHttpBinding.ParsePorts(httpsPorts, isSecure: true),
        ];
        if (existingBindings.Count > 0)
            Log.Information(
                "Dedicated h2c listener(s) enabled; re-binding the HTTP_PORTS surface ({HttpPorts}/{HttpsPorts}) " +
                "alongside the dedicated port(s).", httpPorts, httpsPorts);
    }

    // A driver-only Windows-service node gets NO URLS and NO *_PORTS (Install-Services.ps1 sets URLS
    // only under $hasAdmin), so "nothing configured" is a real, supported state — not an error.
    // Kestrel's own default is localhost:5000; re-state it explicitly, because taking over
    // configuration means taking over the default too. Health probes hit this port.
    if (existingBindings.Count == 0)
    {
        existingBindings = [new KestrelHttpBinding("localhost", 5000, IsSecure: false)];
        Log.Information(
            "Dedicated h2c listener(s) enabled with no URLs configured; re-binding Kestrel's default " +
            "http://localhost:5000 alongside the dedicated port(s).");
    }

    // Re-binding an https endpoint would need its certificate configuration replayed too, which
    // this host does not model (TLS is terminated at Traefik). Rather than silently serve it
    // without TLS — or drop it — refuse to take over Kestrel at all and leave the existing
    // surface exactly as it was. The operator gets a loud, actionable error instead of an
    // AdminUI that stopped answering. Disables BOTH dedicated ports (either would force the same
    // Kestrel takeover).
    if (existingBindings.Any(b => b.IsSecure))
    {
        Log.Error(
            "A dedicated h2c listener is requested (LocalDb:SyncListenPort={SyncPort}, " +
            "ConfigServe:GrpcListenPort={ConfigServePort}, Telemetry:GrpcListenPort={TelemetryPort}) but this " +
            "host serves HTTPS endpoint(s) ({Urls}). " +
            "Binding a dedicated listener requires re-binding every existing endpoint explicitly, and the " +
            "HTTPS certificate configuration cannot be replayed safely. ALL dedicated listeners are DISABLED; " +
            "terminate TLS upstream (as the docker-dev rig does) or leave these features off on this node.",
            syncListenPort, configServeGrpcPort, telemetryListenPort, configuredUrls);
        syncListenPort = 0;
        configServeGrpcPort = 0;
        telemetryListenPort = 0;
    }
    else
    {
        // Capture locals so the ConfigureKestrel closure does not observe later reassignment.
        var syncPortToBind = syncListenPort;
        var configServePortToBind = configServeGrpcPort;
        var telemetryPortToBind = telemetryListenPort;
        builder.WebHost.ConfigureKestrel(kestrel =>
        {
            foreach (var binding in existingBindings)
                binding.Apply(kestrel);

            if (syncPortToBind > 0)
                kestrel.ListenAnyIP(syncPortToBind, o => o.Protocols = HttpProtocols.Http2);
            if (configServePortToBind > 0)
                kestrel.ListenAnyIP(configServePortToBind, o => o.Protocols = HttpProtocols.Http2);
            if (telemetryPortToBind > 0)
                kestrel.ListenAnyIP(telemetryPortToBind, o => o.Protocols = HttpProtocols.Http2);
        });

        Log.Information(
            "Dedicated h2c listener(s) bound (sync=:{SyncPort}, config-serve=:{ConfigServePort}, " +
            "telemetry=:{TelemetryPort}); re-bound existing endpoint(s) {Urls}.",
            syncListenPort, configServeGrpcPort, telemetryListenPort,
            configuredUrls ?? "http://localhost:5000 (Kestrel default)");
    }
}

var app = builder.Build();

// AddZbSerilog registers Serilog as the MEL logging provider but does NOT assign the static
// Serilog.Log.Logger. Set it from the DI root logger so (1) static Log.* calls like the startup
// banner below emit, and (2) Akka.Logger.Serilog's SerilogLogger — which writes to Log.Logger —
// routes the actor graph's logs (DriverHostActor et al.) to the configured sinks. Must run before
// app.RunAsync() starts the ActorSystem (the Akka logger captures Log.Logger at system start).
Serilog.Log.Logger = app.Services.GetRequiredService<Serilog.ILogger>();

app.UseSerilogRequestLogging();

// Razor class library static assets (_content/<libname>/...) are served via endpoint
// routing, NOT the UseStaticFiles middleware — so we MUST mark the static-asset
// endpoints AllowAnonymous, otherwise the AddOtOpcUaAuth fallback RequireAuthenticatedUser
// policy 401s every CSS/JS request and the login page renders unstyled.
app.MapStaticAssets().AllowAnonymous();

if (hasAdmin)
{
    app.UseAuthentication();
    app.UseAuthorization();
    app.UseAntiforgery();
    app.MapOtOpcUaAuth();
    app.MapAdminUI<App>();
    app.MapScriptAnalysisEndpoints();
    app.MapOtOpcUaHubs();
    // Headless deploy trigger for CI/scripts (API-key gated; disabled until Security:DeployApiKey set).
    app.MapOtOpcUaDeployApi(app.Configuration);
}

// Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a
// default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be safe
// even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey configured — but
// not mapping it at all is the stronger default.
if (hasDriver && syncListenPort > 0)
{
    app.MapZbLocalDbSync();
}

// Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on admin-role
// nodes (which hold the central SQL connection) and only when a dedicated h2c port is bound. The
// ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes with no ConfigServe:ApiKey.
if (hasAdmin && configServeGrpcPort > 0)
{
    app.MapGrpcService<DeploymentArtifactGrpcService>();
}

// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
if (hasDriver && telemetryListenPort > 0)
{
    app.MapGrpcService<TelemetryStreamGrpcService>();
}

app.MapOtOpcUaHealth();
app.MapOtOpcUaMetrics();

Log.Information("OtOpcUa.Host starting with roles=[{Roles}] (admin={HasAdmin}, driver={HasDriver})",
    string.Join(",", roles), hasAdmin, hasDriver);

await app.RunAsync();

namespace ZB.MOM.WW.OtOpcUa.Host
{
    /// <summary>Re-exported for <c>WebApplicationFactory&lt;Program&gt;</c> integration tests (F1).</summary>
    public partial class Program;
}