Joseph Doherty
0e162cb250
Caught on the Phase 1A rig proof: central-a logged only "Now listening on: http://[::]:8083" and nothing else. Central UI, the Management + Inbound API, and the /health/* endpoints Traefik + IActiveNodeGate depend on were all gone, with no startup error — the node booted, joined the cluster and served gRPC fine. Cause: calling options.ListenAnyIP in ConfigureKestrel puts Kestrel into explicit-endpoints mode, which SUPPRESSES the URLs from ASPNETCORE_URLS/--urls entirely — it is not additive, contrary to the comment T1A.2 shipped. Central's whole HTTP/1 surface lives on that URL (http://+:5000 on the rig, a different port in prod), so binding only the gRPC port silently deleted it. The site branch has the same shape but no ASPNETCORE_URLS surface to lose — it binds every port it needs explicitly. Fix: parse the port(s) from the configured URLs and re-declare them (Http1AndHttp2) alongside the gRPC port (Http2) in the one ConfigureKestrel call. New Program.ParseHttpBindPorts + CentralHttpBindPortsTests (14 cases: wildcard/ipv6/ hostname hosts, multi-URL, de-dupe, scheme-default, null/blank, unparseable-skipped). Why no test caught it originally: unit/E2E tests use TestServer, which never binds real Kestrel. Only a live node exposes the missing listener — which is exactly what the rig proof is for.
726 lines
42 KiB
C#
726 lines
42 KiB
C#
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.DependencyInjection.Extensions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.Auth.ApiKeys.DependencyInjection;
|
|
using ZB.MOM.WW.Auth.AspNetCore;
|
|
using ZB.MOM.WW.Health;
|
|
using ZB.MOM.WW.Health.Akka;
|
|
using ZB.MOM.WW.Health.EntityFrameworkCore;
|
|
using ZB.MOM.WW.LocalDb.Replication;
|
|
using ZB.MOM.WW.ScadaBridge.AuditLog;
|
|
using ZB.MOM.WW.ScadaBridge.CentralUI;
|
|
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
|
using ZB.MOM.WW.ScadaBridge.Communication;
|
|
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
|
using ZB.MOM.WW.ScadaBridge.DeploymentManager;
|
|
using ZB.MOM.WW.ScadaBridge.ExternalSystemGateway;
|
|
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
using ZB.MOM.WW.ScadaBridge.Host;
|
|
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.Host.Health;
|
|
using ZB.MOM.WW.ScadaBridge.InboundAPI;
|
|
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
|
|
using ZB.MOM.WW.ScadaBridge.KpiHistory;
|
|
using ZB.MOM.WW.ScadaBridge.ManagementService;
|
|
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
|
|
using ZB.MOM.WW.ScadaBridge.NotificationService;
|
|
using ZB.MOM.WW.ScadaBridge.Security;
|
|
using ZB.MOM.WW.ScadaBridge.SiteCallAudit;
|
|
using ZB.MOM.WW.ScadaBridge.TemplateEngine;
|
|
using ZB.MOM.WW.ScadaBridge.Transport;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
using ZB.MOM.WW.Secrets.Configuration;
|
|
using ZB.MOM.WW.Secrets.DependencyInjection;
|
|
using ZB.MOM.WW.Secrets.Sqlite;
|
|
using ZB.MOM.WW.Secrets.Ui;
|
|
using ZB.MOM.WW.Telemetry;
|
|
using Serilog;
|
|
|
|
// SCADABRIDGE_CONFIG determines which role-specific config to load (Central or Site)
|
|
// DOTNET_ENVIRONMENT/ASPNETCORE_ENVIRONMENT stay as "Development" for dev tooling (static assets, EF migrations, etc.)
|
|
var scadabridgeConfig = Environment.GetEnvironmentVariable("SCADABRIDGE_CONFIG")
|
|
?? Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT")
|
|
?? "Production";
|
|
|
|
var configuration = new ConfigurationBuilder()
|
|
.AddJsonFile("appsettings.json", optional: false)
|
|
.AddJsonFile($"appsettings.{scadabridgeConfig}.json", optional: true)
|
|
.AddEnvironmentVariables()
|
|
.AddCommandLine(args)
|
|
.Build();
|
|
|
|
// Expand ${secret:...} config references before any validator/binder sees them (Layer A).
|
|
// Throwaway provider — disposed here, shares no singletons with the host container.
|
|
#pragma warning disable ASP0000 // deliberate throwaway container
|
|
await using (var secretsProvider = new ServiceCollection()
|
|
.AddZbSecrets(configuration, "Secrets")
|
|
.BuildServiceProvider())
|
|
#pragma warning restore ASP0000
|
|
{
|
|
await secretsProvider.GetRequiredService<SqliteSecretsStoreMigrator>().MigrateAsync(default);
|
|
var resolver = secretsProvider.GetRequiredService<ISecretResolver>();
|
|
await new SecretReferenceExpander(resolver)
|
|
.ExpandConfigurationAsync(configuration, default);
|
|
}
|
|
|
|
// Full startup validation — fail fast before any DI or actor system setup
|
|
StartupValidator.Validate(configuration);
|
|
|
|
// Read node options for Serilog enrichment
|
|
var nodeRole = configuration["ScadaBridge:Node:Role"]!;
|
|
var nodeHostname = configuration["ScadaBridge:Node:NodeHostname"] ?? "unknown";
|
|
var siteId = configuration["ScadaBridge:Node:SiteId"] ?? "central";
|
|
|
|
// Serilog structured logging.
|
|
// Minimum level is driven by ScadaBridge:Logging:MinimumLevel (LoggingOptions).
|
|
// Console and file sinks are defined in the `Serilog` configuration
|
|
// section (appsettings.json) and applied via ReadFrom.Configuration inside the
|
|
// factory — the sink set, output template, file path and rolling interval are all
|
|
// configuration-driven, not hard-coded here.
|
|
Log.Logger = ZB.MOM.WW.ScadaBridge.Host.LoggerConfigurationFactory
|
|
.Build(configuration, nodeRole, siteId, nodeHostname)
|
|
.CreateLogger();
|
|
|
|
try
|
|
{
|
|
Log.Information("Starting ScadaBridge host as {Role} on {Hostname}", nodeRole, nodeHostname);
|
|
|
|
if (nodeRole.Equals("Central", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var builder = WebApplication.CreateBuilder(args);
|
|
builder.Configuration.AddConfiguration(configuration);
|
|
|
|
// Serilog
|
|
builder.Host.UseSerilog();
|
|
|
|
// Windows Service support (no-op when not running as a Windows Service)
|
|
builder.Host.UseWindowsService();
|
|
|
|
// Explicit Kestrel h2c listener for the central-hosted gRPC control plane
|
|
// (CentralControlService): HTTP/2-only, on its own port (default 8083, symmetric
|
|
// with the site GrpcPort — a node is either central or a site, never both). gRPC
|
|
// does NOT go through Traefik (HTTP/1 only); sites reach this port by container name.
|
|
//
|
|
// WARNING — the trap the rig caught (T1A.2 shipped it, fixed here): calling
|
|
// options.Listen*/ListenAnyIP puts Kestrel into EXPLICIT-ENDPOINTS mode, which
|
|
// SUPPRESSES the URLs from ASPNETCORE_URLS/--urls entirely — it is NOT additive.
|
|
// Central's ENTIRE HTTP/1 surface (Central UI, Management + Inbound API, and the
|
|
// /health/* endpoints Traefik + IActiveNodeGate depend on) lives on that URL
|
|
// (http://+:5000 on the rig, a different port in production). If we bind only the
|
|
// gRPC port, :5000 vanishes and central is reachable over gRPC while the UI, the
|
|
// management API and health checks are all dead — with no startup error. Unit tests
|
|
// use TestServer and never bind real Kestrel, so only a live node exposes this.
|
|
// The site branch has the same ConfigureKestrel shape but no ASPNETCORE_URLS surface
|
|
// to lose (it binds every port it needs — gRPC + metrics — explicitly). Central must
|
|
// therefore RE-BIND its HTTP port(s) here alongside the gRPC port.
|
|
var centralGrpcPort = configuration.GetValue<int>("ScadaBridge:Node:CentralGrpcPort", 8083);
|
|
// "urls" is WebHostDefaults.ServerUrlsKey — the host setting ASPNETCORE_URLS/--urls
|
|
// populate. Read it as a literal so this needs no extra Hosting using.
|
|
var httpUrls = configuration["ASPNETCORE_URLS"]
|
|
?? builder.WebHost.GetSetting("urls")
|
|
?? "http://+:5000";
|
|
var httpPorts = ParseHttpBindPorts(httpUrls);
|
|
builder.WebHost.ConfigureKestrel(options =>
|
|
{
|
|
// The HTTP/1.1 (+ HTTP/2) surface from the configured URLs, re-declared so it
|
|
// survives the explicit-endpoints switch above.
|
|
foreach (var httpPort in httpPorts)
|
|
{
|
|
options.ListenAnyIP(httpPort, listenOptions =>
|
|
{
|
|
listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http1AndHttp2;
|
|
});
|
|
}
|
|
|
|
// The gRPC control plane, HTTP/2 h2c only, on its own port.
|
|
options.ListenAnyIP(centralGrpcPort, listenOptions =>
|
|
{
|
|
listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2;
|
|
});
|
|
});
|
|
|
|
// Shared components
|
|
builder.Services.AddClusterInfrastructure();
|
|
builder.Services.AddCommunication();
|
|
|
|
// Per-site gRPC preshared keys. Central-only: it is the side that dials sites, and
|
|
// the only side whose key set is dynamic (sites come from the configuration
|
|
// database, so there is no fixed list of ${secret:} references to expand at boot —
|
|
// hence a runtime resolver rather than the pre-host SecretReferenceExpander a site
|
|
// node uses for its single key). Registered before the clients that consume it.
|
|
builder.Services.AddSingleton<
|
|
ZB.MOM.WW.ScadaBridge.Communication.Grpc.ISitePskProvider, SitePskProvider>();
|
|
|
|
// Central-hosted gRPC control plane (T1A.2). CentralControlAuthInterceptor is the
|
|
// per-site sibling of the site's ControlPlaneAuthInterceptor: it verifies the Bearer
|
|
// token against the key for the site named in the required x-scadabridge-site header,
|
|
// resolved through the ISitePskProvider registered just above. Registered BY TYPE on
|
|
// AddGrpc exactly as the site branch registers its interceptors — NOT as a DI singleton,
|
|
// which would let DI hand the instance back and bypass Grpc.AspNetCore's own activation
|
|
// path (the shape that once hid a two-public-constructor defect until the rig caught it).
|
|
// CentralControlGrpcService decodes each request onto the same in-process message the
|
|
// ClusterClient path carries and Asks CentralCommunicationActor — zero handler logic here.
|
|
builder.Services.AddGrpc(options =>
|
|
{
|
|
options.Interceptors.Add<CentralControlAuthInterceptor>();
|
|
});
|
|
builder.Services.AddSingleton<
|
|
ZB.MOM.WW.ScadaBridge.Communication.Grpc.CentralControlGrpcService>();
|
|
builder.Services.AddHealthMonitoring();
|
|
builder.Services.AddCentralHealthAggregation();
|
|
builder.Services.AddExternalSystemGateway();
|
|
builder.Services.AddNotificationService();
|
|
|
|
// Central-only components
|
|
// Notification Outbox: central owns SMTP delivery; the Email adapter reuses the
|
|
// AddNotificationService() SMTP machinery above. AddNotificationOutbox binds
|
|
// NotificationOutboxOptions via BindConfiguration, so no explicit Configure is needed.
|
|
builder.Services.AddNotificationOutbox();
|
|
// Transport — central-only bundle export/import pipeline. Binds
|
|
// TransportOptions from ScadaBridge:Transport via BindConfiguration; no
|
|
// explicit Configure needed.
|
|
builder.Services.AddTransport();
|
|
// Script-artifact invalidation bus: in-process, per-node pub/sub
|
|
// for ScriptArtifactsChanged. The Transport bundle importer publishes
|
|
// post-commit; the Inbound API's ScriptArtifactChangeSubscriber (a hosted
|
|
// service registered by AddInboundAPI below) consumes ApiMethod notifications
|
|
// and invalidates the compiled-handler cache (wired in plan R2-06 T1; the
|
|
// per-request revision check remains the correctness fallback).
|
|
// SharedScript/Template notifications currently have NO consumer — nothing at
|
|
// central caches compiled artifacts of those kinds. Central-only.
|
|
builder.Services.AddSingleton<
|
|
ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Scripting.IScriptArtifactChangeBus,
|
|
ZB.MOM.WW.ScadaBridge.Host.Services.InProcessScriptArtifactChangeBus>();
|
|
// Audit Log — central node owns the AuditLogIngestActor singleton +
|
|
// IAuditLogRepository. The site writer chain is still registered (lazy
|
|
// singletons) but is never resolved on a central node.
|
|
builder.Services.AddAuditLog(builder.Configuration);
|
|
// Central-only hosted service that rolls
|
|
// pf_AuditLog_Month forward monthly. Depends on IPartitionMaintenance
|
|
// (registered below by AddConfigurationDatabase).
|
|
builder.Services.AddAuditLogCentralMaintenance(builder.Configuration);
|
|
// Central-only registration backing the two
|
|
// maintenance singletons started in AkkaHostedService: the production
|
|
// ISiteEnumerator + IPullAuditEventsClient (gRPC) used by the
|
|
// SiteAuditReconciliationActor, plus the AuditLogPurgeOptions /
|
|
// SiteAuditReconciliationOptions bindings consumed by both singletons.
|
|
// Central-only by design (it dials sites), kept out of AddAuditLog.
|
|
builder.Services.AddAuditLogCentralReconciliationClient(builder.Configuration);
|
|
// Site Call Audit — central node owns the SiteCallAuditActor
|
|
// singleton. The extension itself currently registers
|
|
// nothing — actor Props are constructed inline in AkkaHostedService —
|
|
// but the call is here for symmetry with the other audit composition
|
|
// roots so future per-actor DI lands without touching Program.cs.
|
|
builder.Services.AddSiteCallAudit();
|
|
// KPI History — central-only. Binds KpiHistoryOptions from
|
|
// ScadaBridge:KpiHistory and registers the validated options consumed by
|
|
// the KpiHistoryRecorderActor cluster singleton (started in
|
|
// AkkaHostedService). Observability/best-effort: NOT readiness-gated.
|
|
builder.Services.AddKpiHistory(builder.Configuration);
|
|
builder.Services.AddTemplateEngine();
|
|
builder.Services.AddDeploymentManager();
|
|
// Host is the composition root and owns config-coupled wiring: register the
|
|
// shared LDAP auth (binds LdapOptions + IValidateOptions<LdapOptions> with
|
|
// ValidateOnStart + ILdapAuthService singleton) here, then AddSecurity() for the
|
|
// config-free remainder. AddSecurity is a component library and takes no
|
|
// IConfiguration (Options pattern only). Behaviour-preserving: identical
|
|
// registrations to the previous AddSecurity(builder.Configuration) call.
|
|
builder.Services.AddZbLdapAuth(
|
|
builder.Configuration,
|
|
ZB.MOM.WW.ScadaBridge.Security.ServiceCollectionExtensions.LdapSectionPath);
|
|
// Dev disable-login flag (config-coupled, so read + bound here at the composition root,
|
|
// mirroring AddZbLdapAuth). Default false. See AuthDisableLoginOptions / disable-login design doc.
|
|
builder.Services.AddOptions<ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions>()
|
|
.Bind(builder.Configuration.GetSection(
|
|
ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.SectionName));
|
|
var disableLogin = builder.Configuration
|
|
.GetSection(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.SectionName)
|
|
.GetValue<bool>(nameof(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.DisableLogin));
|
|
// Fail-fast guard: DisableLogin outside Development requires the explicit
|
|
// AllowOutsideDevelopment acknowledgement key. See DisableLoginGuard.
|
|
ZB.MOM.WW.ScadaBridge.Security.Auth.DisableLoginGuard.Validate(
|
|
disableLogin,
|
|
builder.Environment.EnvironmentName,
|
|
builder.Configuration
|
|
.GetSection(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.SectionName)
|
|
.GetValue<bool>(nameof(ZB.MOM.WW.ScadaBridge.Security.Auth.AuthDisableLoginOptions.AllowOutsideDevelopment)));
|
|
builder.Services.AddSecurity(disableLogin);
|
|
builder.Services.AddCentralUI();
|
|
// Local SQLite store by default; a local store replicating against a shared SQL-Server hub
|
|
// only when Secrets:Replication:Enabled is true AND a hub connection string is present.
|
|
// See SecretsRegistration for why the gate exists (eager options validation) and why the
|
|
// hub connection string can never itself come from the hub.
|
|
builder.Services.AddScadaBridgeSecrets(builder.Configuration);
|
|
// Secrets UI authorization: adds the named policies secrets:manage + secrets:reveal
|
|
// (role-based) consumed by the /admin/secrets page. AddSecretsAuthorization only ADDS
|
|
// these two policies via Configure<AuthorizationOptions> — it composes additively with
|
|
// AddScadaBridgeAuthorization (invoked inside AddSecurity above) without clobbering.
|
|
// The library's AdminRole ("Administrator") satisfies both policies, and ScadaBridge
|
|
// builds identities with RoleClaimType = ZbClaimTypes.Role, so an Administrator is
|
|
// authorized for both manage + reveal. Done Host-side to keep Secrets.Ui out of Security.
|
|
builder.Services.Configure<AuthorizationOptions>(o => o.AddSecretsAuthorization());
|
|
// Secrets.Ui components audit through the SHARED ZB.MOM.WW.Audit.IAuditWriter seam,
|
|
// which is deliberately distinct from the repo's own Commons IAuditWriter — bridge it
|
|
// to the central direct-write path (ICentralAuditWriter → dbo.AuditLog), NOT the site
|
|
// SQLite chain, which is never resolved on central and has no forwarder here
|
|
// (ScadaBridge#22; component activation 500'd without this registration).
|
|
builder.Services.AddSingleton<ZB.MOM.WW.Audit.IAuditWriter>(
|
|
sp => new ZB.MOM.WW.ScadaBridge.Host.Services.CentralSharedSeamAuditWriter(
|
|
sp.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ICentralAuditWriter>()));
|
|
builder.Services.AddInboundAPI();
|
|
|
|
// Inbound-API auth re-arch: the shared ZB.MOM.WW.Auth.ApiKeys verifier +
|
|
// SQLite store + startup migration are now the SOLE inbound-API auth path.
|
|
// The POST /api/{methodName} endpoint authenticates Bearer tokens
|
|
// (sbk_<keyId>_<secret>) and authorizes by scope == method name through this
|
|
// verifier. The legacy peppered-HMAC X-API-Key path — the SQL Server ApiKey
|
|
// entity, ApiKeyValidator, and IApiKeyHasher — was retired; the
|
|
// ScadaBridge:InboundApi:ApiKeyPepper config key is now consumed only as the
|
|
// library verifier's pepper secret (PepperSecretName below).
|
|
//
|
|
// ApiKeyOptions is an init-only record, so the contract-mandated values
|
|
// are injected as in-memory configuration UNDER the bound section path
|
|
// (ScadaBridge:InboundApi:ApiKeyStore) rather than mutated post-bind:
|
|
// - TokenPrefix = "sbk" (the inbound token prefix)
|
|
// - PepperSecretName points at the EXISTING inbound-API pepper config key
|
|
// (reused so one secret backs both the legacy and new path during the
|
|
// additive window)
|
|
// - RunMigrationsOnStartup = true (hosted service creates the schema)
|
|
// - SqlitePath defaults under the content root's data/ directory, but any
|
|
// value already supplied via appsettings/env wins (AddInMemoryCollection
|
|
// is registered last, but only fills keys the operator did not set
|
|
// because we read the existing value first).
|
|
const string apiKeyStoreSection = "ScadaBridge:InboundApi:ApiKeyStore";
|
|
var configuredSqlitePath = builder.Configuration[$"{apiKeyStoreSection}:SqlitePath"];
|
|
var apiKeyStoreDefaults = new Dictionary<string, string?>
|
|
{
|
|
[$"{apiKeyStoreSection}:TokenPrefix"] = "sbk",
|
|
[$"{apiKeyStoreSection}:PepperSecretName"] = "ScadaBridge:InboundApi:ApiKeyPepper",
|
|
[$"{apiKeyStoreSection}:RunMigrationsOnStartup"] = "true",
|
|
[$"{apiKeyStoreSection}:SqlitePath"] = string.IsNullOrWhiteSpace(configuredSqlitePath)
|
|
? Path.Combine(builder.Environment.ContentRootPath, "data", "inbound-api-keys.sqlite")
|
|
: configuredSqlitePath,
|
|
};
|
|
builder.Configuration.AddInMemoryCollection(apiKeyStoreDefaults);
|
|
|
|
builder.Services.AddZbApiKeyAuth(builder.Configuration, apiKeyStoreSection);
|
|
|
|
// Inbound-API key re-arch, additive: expose the library admin facade
|
|
// (ApiKeyAdminCommands) and the app-side management seam (IInboundApiKeyAdmin)
|
|
// in the SAME container as AddZbApiKeyAuth, so CLI + CentralUI later create /
|
|
// list / enable / disable / delete inbound keys and edit their method-scopes
|
|
// through one shared path. AddZbApiKeyAuth registers the stores/pepper/migrator
|
|
// but NOT ApiKeyAdminCommands itself, so it is composed here. CentralUI resolves
|
|
// from this same provider (it is registered via AddCentralUI() above), so the
|
|
// seam is reachable from both the ManagementActor and CentralUI pages — exactly
|
|
// as IInboundApiRepository already is.
|
|
builder.Services.AddSingleton(sp => new ZB.MOM.WW.Auth.ApiKeys.Admin.ApiKeyAdminCommands(
|
|
sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyOptions>>().Value,
|
|
sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyAdminStore>(),
|
|
sp.GetRequiredService<ZB.MOM.WW.Auth.Abstractions.ApiKeys.IApiKeyAuditStore>(),
|
|
sp.GetRequiredService<ZB.MOM.WW.Auth.ApiKeys.IApiKeyPepperProvider>(),
|
|
sp.GetRequiredService<ZB.MOM.WW.Auth.ApiKeys.Sqlite.SqliteAuthStoreMigrator>(),
|
|
TimeProvider.System));
|
|
builder.Services.AddSingleton<
|
|
ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Security.IInboundApiKeyAdmin,
|
|
LibraryInboundApiKeyAdmin>();
|
|
|
|
builder.Services.AddManagementService();
|
|
|
|
var configDbConnectionString = configuration["ScadaBridge:Database:ConfigurationDb"]
|
|
?? throw new InvalidOperationException("ScadaBridge:Database:ConfigurationDb connection string is required for Central role.");
|
|
builder.Services.AddConfigurationDatabase(configDbConnectionString);
|
|
|
|
// Health checks for readiness gating — shared ZB.MOM.WW.Health probes.
|
|
// Ready tier (/health/ready): database + akka-cluster + required-singletons.
|
|
// Active tier (/health/active): active-node = oldest Up member (singleton host) AND
|
|
// database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
|
|
// rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
|
|
// NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
|
|
// placement after a restart and both sides claim leadership during a partition).
|
|
// The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
|
|
// the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
|
|
builder.Services.AddHealthChecks()
|
|
.AddTypeActivatedCheck<DatabaseHealthCheck<ScadaBridgeDbContext>>(
|
|
"database",
|
|
failureStatus: null,
|
|
tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })
|
|
.AddTypeActivatedCheck<AkkaClusterHealthCheck>(
|
|
"akka-cluster",
|
|
failureStatus: null,
|
|
tags: new[] { ZbHealthTags.Ready },
|
|
args: AkkaClusterStatusPolicy.Default)
|
|
// Readiness ALSO reflects "required cluster singletons running"
|
|
// (REQ-HOST-4a). Probes each central singleton's local ClusterSingletonProxy
|
|
// with a bounded Identify and degrades to Unhealthy if any required singleton
|
|
// is unreachable. Registered inside the Central-role branch (this is it) so the
|
|
// check is naturally role-scoped — site nodes never run it. It resolves
|
|
// ActorSystem from DI per probe, like the akka-cluster check above, and is
|
|
// leadership-agnostic so a ready standby still reports ready (the proxy reaches
|
|
// the singleton from either node).
|
|
.AddTypeActivatedCheck<RequiredSingletonsHealthCheck>(
|
|
"required-singletons",
|
|
failureStatus: null,
|
|
tags: new[] { ZbHealthTags.Ready })
|
|
// Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
|
|
// Resolves ActorSystem from DI, like the akka-cluster check above.
|
|
.AddTypeActivatedCheck<OldestNodeActiveHealthCheck>(
|
|
"active-node",
|
|
failureStatus: null,
|
|
tags: new[] { ZbHealthTags.Active });
|
|
|
|
// Akka.NET bootstrap via hosted service
|
|
builder.Services.AddSingleton<AkkaHostedService>();
|
|
builder.Services.AddHostedService(sp => sp.GetRequiredService<AkkaHostedService>());
|
|
|
|
// Bridge the AkkaHostedService-owned ActorSystem to DI as a SINGLETON via
|
|
// GetOrCreateActorSystem(). The shared ZB.MOM.WW.Health Akka checks resolve ActorSystem
|
|
// from DI, per probe, inside a child scope. ActorSystem is IDisposable, so a TRANSIENT
|
|
// (or scoped) bridge is captured-and-disposed by each probe's scope — disposing the live
|
|
// system mid-flight (CoordinatedShutdown/ActorSystemTerminateReason) and wedging the
|
|
// central report pages at the 30s Ask timeout. A singleton is resolved from the root and
|
|
// never disposed by a child scope; routing through GetOrCreateActorSystem (instead of a
|
|
// plain singleton factory over .ActorSystem) means the first resolve CREATES the system
|
|
// rather than caching a null if a probe wins the startup race.
|
|
builder.Services.AddSingleton<Akka.Actor.ActorSystem>(sp =>
|
|
sp.GetRequiredService<AkkaHostedService>().GetOrCreateActorSystem());
|
|
|
|
// Register the production IActiveNodeGate implementation so
|
|
// standby-node gating is actually enforced (the InboundApiEndpointFilter
|
|
// consults IActiveNodeGate and defaults to "allow" when none is registered,
|
|
// which leaves the design's "central cluster only (active node)" guarantee
|
|
// unenforced in deployed binaries). The gate is backed by the same oldest-member
|
|
// evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
|
|
// inbound API and the /health/active endpoint Traefik routes against agree on
|
|
// which node is active.
|
|
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.InboundAPI.IActiveNodeGate, ActiveNodeGate>();
|
|
|
|
// Admin-triggered manual failover of the central pair (Health page control,
|
|
// decision 2026-07-22). Central-only: the seam is declared in CentralUI so that
|
|
// project stays Akka-free, and only this branch has a cluster to act on.
|
|
builder.Services.AddSingleton<
|
|
ZB.MOM.WW.ScadaBridge.CentralUI.Services.IManualFailoverService, AkkaManualFailoverService>();
|
|
|
|
// Cluster node status provider scoped to the Central role — feeds the
|
|
// CentralHealthReportLoop so the central cluster appears on /monitoring/health.
|
|
builder.Services.AddSingleton<IClusterNodeProvider>(sp =>
|
|
{
|
|
var akkaService = sp.GetRequiredService<AkkaHostedService>();
|
|
return new AkkaClusterNodeProvider(akkaService, "Central");
|
|
});
|
|
|
|
// Options binding
|
|
SiteServiceRegistration.BindSharedOptions(builder.Services, builder.Configuration);
|
|
builder.Services.Configure<SecurityOptions>(builder.Configuration.GetSection("ScadaBridge:Security"));
|
|
builder.Services.AddOptions<InboundApiOptions>()
|
|
.Bind(builder.Configuration.GetSection("ScadaBridge:InboundApi"))
|
|
.ValidateOnStart();
|
|
builder.Services.TryAddEnumerable(
|
|
ServiceDescriptor.Singleton<IValidateOptions<InboundApiOptions>, InboundApiOptionsValidator>());
|
|
builder.Services.Configure<DeploymentManagerOptions>(
|
|
builder.Configuration.GetSection(ZB.MOM.WW.ScadaBridge.DeploymentManager.ServiceCollectionExtensions.OptionsSection));
|
|
|
|
var app = builder.Build();
|
|
|
|
// Apply or validate database migrations (skip when running in test harness)
|
|
if (!string.Equals(configuration["ScadaBridge:Database:SkipMigrations"], "true", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var isDevelopment = app.Environment.IsDevelopment()
|
|
|| string.Equals(Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"), "Development", StringComparison.OrdinalIgnoreCase);
|
|
var migrationLogger = app.Services
|
|
.GetRequiredService<ILoggerFactory>()
|
|
.CreateLogger(typeof(MigrationHelper).FullName!);
|
|
|
|
// Tolerate a database that is briefly unreachable at boot
|
|
// (e.g. app and DB containers starting together) with a bounded
|
|
// exponential backoff before failing fatally.
|
|
// Only connection-class (transient) faults are retried — a
|
|
// schema-version mismatch is permanent and must fail fast on attempt 1.
|
|
// Thread the host's ApplicationStopping token into both the
|
|
// migration call itself and the inter-attempt Task.Delay so a SIGTERM
|
|
// during the bounded-retry window (~2 min worst-case) tears down
|
|
// cleanly instead of being ignored until the loop exhausts.
|
|
await StartupRetry.ExecuteWithRetryAsync(
|
|
"database-migration",
|
|
async ct =>
|
|
{
|
|
using var scope = app.Services.CreateScope();
|
|
var dbContext = scope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>();
|
|
await MigrationHelper.ApplyOrValidateMigrationsAsync(dbContext, isDevelopment, migrationLogger, ct);
|
|
},
|
|
maxAttempts: 8,
|
|
initialDelay: TimeSpan.FromSeconds(2),
|
|
migrationLogger,
|
|
isTransient: StartupRetry.IsTransientDatabaseFault,
|
|
cancellationToken: app.Lifetime.ApplicationStopping);
|
|
}
|
|
|
|
// Middleware pipeline
|
|
|
|
// Trusted-proxy forwarded headers (arch-review R2 N2): behind Traefik every client
|
|
// shares the proxy's IP, collapsing the LoginThrottle's {username}|{ip} keys onto
|
|
// one address (per-IP isolation gone + cheap username-lockout DoS). When enabled,
|
|
// X-Forwarded-For from the CONFIGURED proxies only is honored. MUST run first —
|
|
// everything downstream keys on Connection.RemoteIpAddress.
|
|
var forwardedOptions = builder.Configuration
|
|
.GetSection(ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSecurityOptions.SectionName)
|
|
.Get<ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSecurityOptions>() ?? new();
|
|
if (ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSetup.Build(forwardedOptions) is { } fho)
|
|
app.UseForwardedHeaders(fho);
|
|
|
|
app.UseWebSockets();
|
|
app.UseRouting();
|
|
app.UseAuthentication();
|
|
app.UseAuthorization();
|
|
app.UseAntiforgery();
|
|
|
|
// Emit one InboundRequest/InboundAuthFailure
|
|
// audit row per call into the inbound API. Placed AFTER UseAuthentication/
|
|
// UseAuthorization so any HttpContext.User the framework populates is in
|
|
// place, and scoped to the /api/ prefix so it never observes the Central UI,
|
|
// Management API, SignalR hubs, or health endpoints. The endpoint handler
|
|
// is responsible for stashing the resolved API key name on
|
|
// HttpContext.Items (see AuditWriteMiddleware.AuditActorItemKey) AFTER its
|
|
// in-handler API key validation succeeds.
|
|
// Scope the audit middleware to the inbound API method
|
|
// route (/api/{methodName}) and explicitly exclude the management/audit
|
|
// sub-trees that share the /api prefix. Without these exclusions the
|
|
// middleware would emit a spurious ApiInbound audit row for every
|
|
// /api/audit/query and /api/audit/export call (and would treat audit-log
|
|
// reads as inbound script invocations — recursive write-on-read). The
|
|
// POST-only filter rules out the GET routes on /api/audit, /api/centralui,
|
|
// /api/script-analysis even if a future route is added under those
|
|
// prefixes with the same verb; the explicit prefix excludes still belt-
|
|
// and-brace POST-y additions there.
|
|
app.UseWhen(
|
|
ctx => ctx.Request.Path.StartsWithSegments("/api")
|
|
&& !ctx.Request.Path.StartsWithSegments("/api/audit")
|
|
&& !ctx.Request.Path.StartsWithSegments("/api/centralui")
|
|
&& !ctx.Request.Path.StartsWithSegments("/api/script-analysis")
|
|
&& !ctx.Request.Path.StartsWithSegments("/api/management")
|
|
&& HttpMethods.IsPost(ctx.Request.Method),
|
|
branch => branch.UseAuditWriteMiddleware());
|
|
|
|
// Map the canonical three-tier health endpoints in one call:
|
|
// /health/ready — Ready-tagged checks (database + akka-cluster + required-singletons).
|
|
// REQ-HOST-4a defines readiness as cluster membership + DB connectivity,
|
|
// explicitly NOT active-node status, so the active-node check is excluded
|
|
// (a fully operational standby central node still reports ready).
|
|
// /health/active — Active-tagged checks (active-node = oldest Up member + database
|
|
// reachable); returns 200 only on the singleton-host node with a live
|
|
// DB; used by Traefik for routing.
|
|
// /healthz — bare process liveness; runs no checks (always 200 while the process
|
|
// is up). New tier added by adopting the shared library.
|
|
// All three are anonymous and use the canonical ZbHealthWriter JSON output.
|
|
app.MapZbHealth();
|
|
|
|
// Observability — mount the always-on Prometheus /metrics scrape endpoint.
|
|
// AddZbTelemetry (in SiteServiceRegistration.BindSharedOptions) wires the OTel
|
|
// Resource + standard instrumentation + Prometheus exporter; this exposes them.
|
|
// Requires endpoint routing (app.UseRouting() above).
|
|
app.MapZbMetrics();
|
|
|
|
// Central-hosted gRPC control plane (T1A.2) — the site→central CentralControlService.
|
|
// Runs on the dedicated h2c listener configured above (default :8083), gated by
|
|
// CentralControlAuthInterceptor and readiness-gated by the service itself (Unavailable
|
|
// until AkkaHostedService hands the CentralCommunicationActor over via SetReady). It
|
|
// shares the app's endpoint routing with the HTTP/1 surface; Kestrel steers each
|
|
// connection to the right pipeline by listener/protocol.
|
|
app.MapGrpcService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.CentralControlGrpcService>();
|
|
|
|
app.MapStaticAssets();
|
|
app.MapCentralUI<ZB.MOM.WW.ScadaBridge.Host.Components.App>();
|
|
app.MapInboundAPI();
|
|
app.MapManagementAPI();
|
|
// Audit Log: CLI-facing /api/audit/{query,export} routes. Same
|
|
// Basic-Auth + LDAP mechanism as /management; gated on the OperationalAudit
|
|
// / AuditExport role sets.
|
|
app.MapAuditAPI();
|
|
// Notify-and-fetch deploy: site-facing token-gated fetch of a staged
|
|
// deployment's flattened config. Machine-to-machine — AllowAnonymous, gated
|
|
// solely by the per-deployment X-Deployment-Token (no central FallbackPolicy).
|
|
app.MapDeploymentConfigAPI();
|
|
app.MapHub<ZB.MOM.WW.ScadaBridge.ManagementService.DebugStreamHub>("/hubs/debug-stream");
|
|
|
|
// Compile and register all Inbound API method scripts at startup
|
|
using (var scope = app.Services.CreateScope())
|
|
{
|
|
var apiRepo = scope.ServiceProvider.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IInboundApiRepository>();
|
|
var executor = app.Services.GetRequiredService<ZB.MOM.WW.ScadaBridge.InboundAPI.InboundScriptExecutor>();
|
|
var methods = await apiRepo.GetAllApiMethodsAsync();
|
|
// Parallel: CompileAndRegister is safe for concurrent callers (ConcurrentDictionary
|
|
// caches, no shared mutable compile state); serial Roslyn compiles stretch the
|
|
// failover-recovery window at hundreds of methods.
|
|
Parallel.ForEach(methods, method => executor.CompileAndRegister(method));
|
|
}
|
|
|
|
await app.RunAsync();
|
|
}
|
|
else if (nodeRole.Equals("Site", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var builder = WebApplication.CreateBuilder(args);
|
|
builder.Configuration.AddConfiguration(configuration);
|
|
|
|
// Serilog
|
|
builder.Host.UseSerilog();
|
|
|
|
// Windows Service support (no-op when not running as a Windows Service)
|
|
builder.Host.UseWindowsService();
|
|
|
|
// Read GrpcPort from config (NodeOptions already has default 8083)
|
|
var grpcPort = configuration.GetValue<int>("ScadaBridge:Node:GrpcPort", 8083);
|
|
|
|
// Read MetricsPort from config (NodeOptions already has default 8084).
|
|
// Separate HTTP/1.1 listener so a standard HTTP/1.1 Prometheus scraper can
|
|
// reach /metrics; the gRPC port stays HTTP/2-only below. The default is
|
|
// 8084 — distinct from RemotingPort (8082, Akka) and GrpcPort (8083) so the
|
|
// metrics listener never collides with the Akka remoting port on site nodes.
|
|
var metricsPort = configuration.GetValue<int>("ScadaBridge:Node:MetricsPort", 8084);
|
|
|
|
// Configure Kestrel for HTTP/2 only on the gRPC port
|
|
builder.WebHost.ConfigureKestrel(options =>
|
|
{
|
|
options.ListenAnyIP(grpcPort, listenOptions =>
|
|
{
|
|
listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2;
|
|
});
|
|
|
|
// Dedicated HTTP/1.1 (and HTTP/2) listener for the Prometheus /metrics
|
|
// scrape endpoint, reachable by an HTTP/1.1 scraper.
|
|
options.ListenAnyIP(metricsPort, listenOptions =>
|
|
{
|
|
listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http1AndHttp2;
|
|
});
|
|
});
|
|
|
|
// gRPC server registration. Two interceptors, two disjoint service prefixes, two
|
|
// separate keys — neither one's absence weakens the other:
|
|
//
|
|
// LocalDbSyncAuthInterceptor gates /localdb_sync.v1.LocalDbSync/ (LocalDb:Replication:ApiKey)
|
|
// ControlPlaneAuthInterceptor gates /sitestream.SiteStreamService/ (ScadaBridge:Communication:GrpcPsk)
|
|
//
|
|
// Both are fail-closed: an unset key closes that surface rather than opening it. For
|
|
// LocalDb that means replication simply does not start; for the control plane it means
|
|
// this node serves no streams and no audit pulls until a key is configured, which is
|
|
// why every environment must carry one before running this build.
|
|
builder.Services.AddGrpc(options =>
|
|
{
|
|
options.Interceptors.Add<LocalDbSyncAuthInterceptor>();
|
|
options.Interceptors.Add<ControlPlaneAuthInterceptor>();
|
|
});
|
|
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
|
|
|
|
// Existing site service registrations (this is also where LocalDb and its
|
|
// replication engine are registered — see SiteServiceRegistration)
|
|
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
|
|
|
|
var app = builder.Build();
|
|
|
|
// Endpoint routing middleware. The gRPC service mapping below and the
|
|
// /metrics scrape endpoint both run on endpoint routing, so UseRouting()
|
|
// must be present before the Map* calls on the site role.
|
|
app.UseRouting();
|
|
|
|
// Observability — mount the always-on Prometheus /metrics scrape endpoint.
|
|
// AddZbTelemetry (in SiteServiceRegistration.Configure → BindSharedOptions)
|
|
// wires the OTel Resource + standard instrumentation + Prometheus exporter;
|
|
// this exposes them on the site node too.
|
|
app.MapZbMetrics();
|
|
|
|
// Map gRPC service — resolves the singleton SiteStreamGrpcServer from DI
|
|
app.MapGrpcService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
|
|
|
|
// The passive half of LocalDb replication: the peer node dials THIS endpoint.
|
|
// It shares the Kestrel h2c listener the site gRPC server already uses, so no
|
|
// listener or port changes are needed. Mapping it is harmless with no peer
|
|
// configured — nothing dials it, and LocalDbSyncAuthInterceptor (registered on
|
|
// AddGrpc above) rejects anything that tries until an ApiKey is configured.
|
|
app.MapZbLocalDbSync();
|
|
|
|
// Site-shutdown ordering. ApplicationStopping
|
|
// fires BEFORE IHostedService.StopAsync runs, so the gRPC server
|
|
// refuses new streams (Unavailable) and cancels every active stream
|
|
// here — clients observe a clean Cancelled and reconnect — and only
|
|
// THEN does AkkaHostedService run CoordinatedShutdown and tear down
|
|
// actors. Without this hand-off, in-flight streams go silent and only
|
|
// time out via gRPC keepalive (~25 s), violating the documented
|
|
// four-step sequence.
|
|
var siteLifetime = app.Services.GetRequiredService<Microsoft.Extensions.Hosting.IHostApplicationLifetime>();
|
|
var siteGrpcServer = app.Services.GetRequiredService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
|
|
siteLifetime.ApplicationStopping.Register(() => siteGrpcServer.CancelAllStreams());
|
|
|
|
await app.RunAsync();
|
|
}
|
|
else
|
|
{
|
|
throw new InvalidOperationException($"Unknown role: {nodeRole}. Must be 'Central' or 'Site'.");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Fatal(ex, "ScadaBridge host terminated unexpectedly");
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
await Log.CloseAndFlushAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Exposes the auto-generated Program class for test infrastructure (e.g. WebApplicationFactory).
|
|
/// </summary>
|
|
public partial class Program
|
|
{
|
|
/// <summary>
|
|
/// Extracts the distinct TCP ports from an ASP.NET Core server-URLs string (the
|
|
/// <c>ASPNETCORE_URLS</c> / <c>--urls</c> value, e.g. <c>"http://+:5000"</c> or a
|
|
/// semicolon-separated list). Used to re-declare central's HTTP surface after the gRPC
|
|
/// listener switches Kestrel into explicit-endpoints mode — see the call site's warning.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Bind hosts (<c>+</c>, <c>*</c>, <c>0.0.0.0</c>, <c>[::]</c>, a hostname) are irrelevant
|
|
/// here because the caller re-binds via <c>ListenAnyIP</c>; only the port matters. A URL
|
|
/// with no explicit port falls back to the scheme default (80/443). Unparseable entries
|
|
/// are skipped rather than throwing — a bad URL should not take the node down at boot.
|
|
/// </remarks>
|
|
/// <param name="serverUrls">The server-URLs string; may be null/empty.</param>
|
|
/// <returns>The distinct ports, in first-seen order.</returns>
|
|
internal static IReadOnlyList<int> ParseHttpBindPorts(string? serverUrls)
|
|
{
|
|
var ports = new List<int>();
|
|
if (string.IsNullOrWhiteSpace(serverUrls))
|
|
{
|
|
return ports;
|
|
}
|
|
|
|
foreach (var raw in serverUrls.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
|
|
{
|
|
int port;
|
|
// Uri can't parse the wildcard hosts Kestrel accepts (+, *), so normalize them
|
|
// to a placeholder host before parsing; the host is discarded anyway.
|
|
var normalized = raw.Replace("://+", "://placeholder").Replace("://*", "://placeholder");
|
|
if (Uri.TryCreate(normalized, UriKind.Absolute, out var uri))
|
|
{
|
|
port = uri.Port; // Uri fills the scheme default (80/443) when none is given.
|
|
}
|
|
else
|
|
{
|
|
// Last-ditch: pull the port after the final ':' (handles odd inputs Uri rejects).
|
|
var colon = raw.LastIndexOf(':');
|
|
if (colon < 0 || !int.TryParse(raw.AsSpan(colon + 1), out port))
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (port is > 0 and <= 65535 && !ports.Contains(port))
|
|
{
|
|
ports.Add(port);
|
|
}
|
|
}
|
|
|
|
return ports;
|
|
}
|
|
}
|