From e9c412e528e1e7541a3706df45156264a01209bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 8 Aug 2026 05:23:26 -0400 Subject: [PATCH] fix(host): unhandled boot exception now kills the process instead of wedging the container (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: dotnet runs as container PID 1 and Linux ignores default-action signals sent to PID 1, so the runtime's unhandled-exception path (banner, then abort() -> SIGABRT) could never terminate the process — it printed the trace and spun the main thread at 100% CPU with the container `running`, so `restart: unless-stopped` never fired. Reproduced deterministically: same StartupValidator throw exits 134 under an init process and wedges without one. Two layers, each covering the other's gap: - Program.cs registers an AppDomain.UnhandledException handler before the first statement that can throw: prints the trace, best-effort flushes Serilog, Environment.Exit(134) — exit() is a syscall PID 1 CAN perform, 134 preserves the 128+SIGABRT crash code, and it covers every thread, not just the boot window. It cannot fire under WebApplicationFactory (the test host catches entry-point exceptions), so the designed boot-refusal exceptions still propagate to tests unchanged. - docker-compose: init: true on all 8 nodes for the crash paths that bypass the managed event (Environment.FailFast, runtime-internal aborts). The CoordinatedShutdown no-Environment.Exit guard gains a precise carve-out (exactly one call, only inside the handler); Environment.Exit still fires the CLR shutdown hook Akka binds via run-by-clr-shutdown-hook = on, so the crash path skips nothing abort() kept. New pin test keeps the handler ahead of the configuration build. Live-verified on the rig image: crash now yields Exited (134) + RestartCount climbing under `unless-stopped`, trace intact, with and without init; full 8-node rig redeployed healthy with docker-init as PID 1. Closes #34. Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1 --- docker/docker-compose.yml | 14 ++++++++++ src/ZB.MOM.WW.ScadaBridge.Host/Program.cs | 25 ++++++++++++++++- .../CoordinatedShutdownTests.cs | 23 +++++++++++++++ .../HostStartupTests.cs | 28 +++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b1c17a85..ac54161e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -60,6 +60,13 @@ x-secrets-hub-site-env: &secrets-hub-site-env services: central-a: image: scadabridge:latest + # An init process (tini) as PID 1, so a crashed dotnet process actually dies. + # Without it dotnet IS PID 1, Linux ignores the SIGABRT the runtime's crash path + # raises against PID 1, and any unhandled boot exception left the container + # `running` with the main thread spinning at 100% CPU — restart policy never + # fired (ScadaBridge#34). Belt to Program.cs's UnhandledException handler, which + # covers managed exceptions but not FailFast/runtime-internal aborts. + init: true # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -116,6 +123,7 @@ services: central-b: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -172,6 +180,7 @@ services: site-a-a: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -195,6 +204,7 @@ services: site-a-b: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -218,6 +228,7 @@ services: site-b-a: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -238,6 +249,7 @@ services: site-b-b: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -258,6 +270,7 @@ services: site-c-a: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). @@ -278,6 +291,7 @@ services: site-c-b: image: scadabridge:latest + init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed # mid-drain, turning every redeploy into the crash path (review 01 [Medium]). diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs index e8ab8291..1f997614 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs @@ -39,6 +39,27 @@ using ZB.MOM.WW.Secrets.Ui; using ZB.MOM.WW.Telemetry; using Serilog; +// Terminate honestly on ANY unhandled exception, from any thread — registered before the +// first statement that can throw. In a container this process runs as PID 1, and Linux +// ignores default-action signals sent to PID 1, so the runtime's crash path (print the +// banner, then abort() → SIGABRT) can never actually kill the process: observed live, it +// printed the banner and then spun the main thread at 100% CPU with the container still +// `running`, so `restart: unless-stopped` never fired and every fail-closed boot refusal +// became a silent manual-restart outage (ScadaBridge#34, reproduced 2026-08-07). +// Environment.Exit uses the exit() syscall, which PID 1 CAN perform, and 134 preserves the +// 128+SIGABRT code the abort would have produced. This never fires under test hosts: +// WebApplicationFactory catches entry-point exceptions itself, so they are never +// "unhandled" there and the designed boot-refusal exceptions still propagate to tests. +// The docker compose additionally sets `init: true` for the crash paths that bypass this +// event entirely (Environment.FailFast, runtime-internal aborts). +AppDomain.CurrentDomain.UnhandledException += (_, e) => +{ + // Print ourselves: exiting here preempts the runtime's own banner. + Console.Error.WriteLine(e.ExceptionObject); + try { Log.CloseAndFlush(); } catch { /* stderr above already has the exception */ } + Environment.Exit(134); +}; + // SCADABRIDGE_CONFIG determines which role-specific config to load (Central or Site) // DOTNET_ENVIRONMENT/ASPNETCORE_ENVIRONMENT stay as "Development" for dev tooling (static assets, EF migrations, etc.) var scadabridgeConfig = Environment.GetEnvironmentVariable("SCADABRIDGE_CONFIG") @@ -68,7 +89,9 @@ var configuration = new ConfigurationBuilder() // Two consequences worth knowing. This block runs BEFORE Serilog exists and outside the // try/catch below, so a SQL Server outage at central boot exits with a bare stderr stack // trace, no structured log — honest and container-restart-retryable, just unenriched; the -// SQLite path had the same window but its only failure mode was a local file. And the +// SQLite path had the same window but its only failure mode was a local file. (That exit +// is real only because of the UnhandledException handler at the top of this file — the +// runtime's own crash path wedges as container PID 1, ScadaBridge#34.) And the // role check is deliberately NOT folded into SecretsRegistration: that class refuses to // read the node role from configuration (a config-read role could silently turn a site // into a hub — see SecretsNodeRole), while Program.cs must read it to branch the whole diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs index 5b1a51ac..2dd5adbf 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs @@ -20,6 +20,29 @@ public class CoordinatedShutdownTests foreach (var file in sourceFiles) { var content = File.ReadAllText(file); + + // Sole permitted call site: Program.cs's AppDomain.UnhandledException handler + // (ScadaBridge#34). That is the crash path — the alternative there was never a + // CoordinatedShutdown but the runtime's abort(), which cannot terminate PID 1 + // and wedged the container; Environment.Exit still fires the CLR shutdown hook + // Akka binds via run-by-clr-shutdown-hook = on, so it skips nothing abort kept. + // Everywhere else the original rule stands: no code path may bypass + // CoordinatedShutdown by exiting directly. + if (Path.GetFileName(file) == "Program.cs") + { + var occurrences = CountOccurrences(content, "Environment.Exit("); + Assert.Equal(1, occurrences); + + var handlerAt = content.IndexOf( + "AppDomain.CurrentDomain.UnhandledException +=", StringComparison.Ordinal); + Assert.True(handlerAt >= 0, "Program.cs must register the UnhandledException handler"); + var handlerEnd = content.IndexOf("};", handlerAt, StringComparison.Ordinal); + var exitAt = content.IndexOf("Environment.Exit(", StringComparison.Ordinal); + Assert.True(exitAt > handlerAt && exitAt < handlerEnd, + "Environment.Exit in Program.cs is only permitted inside the UnhandledException handler"); + continue; + } + Assert.DoesNotContain("Environment.Exit", content, StringComparison.Ordinal); } diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HostStartupTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HostStartupTests.cs index 206926ad..05a43234 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HostStartupTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HostStartupTests.cs @@ -165,6 +165,34 @@ public class HostStartupTests : IDisposable } } + [Fact] + public void Program_RegistersUnhandledExceptionExitHandler() + { + // ScadaBridge#34: in a container this process is PID 1, and Linux ignores the + // SIGABRT the runtime's crash path raises against PID 1 — an unhandled boot + // exception printed its banner and then spun the main thread at 100% CPU with + // the container still `running`, so the restart policy never fired. Program.cs + // therefore registers an AppDomain.UnhandledException handler that exits via + // the exit() syscall (which PID 1 CAN perform) before anything can throw. + // Behavioural coverage needs a real crashed process (see the compose comment + // and the issue's live reproductions); this pins the handler's existence so a + // refactor cannot silently reopen the wedge. + var hostProjectDir = FindHostProjectDirectory(); + Assert.NotNull(hostProjectDir); + + var program = File.ReadAllText(Path.Combine(hostProjectDir!, "Program.cs")); + + Assert.Contains("AppDomain.CurrentDomain.UnhandledException +=", program); + Assert.Contains("Environment.Exit(134)", program); + + // The registration must precede the first statement that can throw — the + // configuration build is the earliest (appsettings.json is non-optional). + var handlerAt = program.IndexOf("AppDomain.CurrentDomain.UnhandledException +=", StringComparison.Ordinal); + var configAt = program.IndexOf("new ConfigurationBuilder()", StringComparison.Ordinal); + Assert.True(handlerAt >= 0 && configAt >= 0 && handlerAt < configAt, + "the UnhandledException exit handler must be registered before the configuration build"); + } + private static string? FindHostProjectDirectory() { // Walk up from the test assembly location to find the src directory