Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HostStartupTests.cs
T
Joseph Doherty e9c412e528 fix(host): unhandled boot exception now kills the process instead of wedging the container (#34)
Root cause: dotnet runs as container PID 1 and Linux ignores default-action
signals sent to PID 1, so the runtime's unhandled-exception path (banner,
then abort() -> SIGABRT) could never terminate the process — it printed the
trace and spun the main thread at 100% CPU with the container `running`,
so `restart: unless-stopped` never fired. Reproduced deterministically:
same StartupValidator throw exits 134 under an init process and wedges
without one.

Two layers, each covering the other's gap:
- Program.cs registers an AppDomain.UnhandledException handler before the
  first statement that can throw: prints the trace, best-effort flushes
  Serilog, Environment.Exit(134) — exit() is a syscall PID 1 CAN perform,
  134 preserves the 128+SIGABRT crash code, and it covers every thread,
  not just the boot window. It cannot fire under WebApplicationFactory
  (the test host catches entry-point exceptions), so the designed
  boot-refusal exceptions still propagate to tests unchanged.
- docker-compose: init: true on all 8 nodes for the crash paths that
  bypass the managed event (Environment.FailFast, runtime-internal aborts).

The CoordinatedShutdown no-Environment.Exit guard gains a precise carve-out
(exactly one call, only inside the handler); Environment.Exit still fires
the CLR shutdown hook Akka binds via run-by-clr-shutdown-hook = on, so the
crash path skips nothing abort() kept. New pin test keeps the handler ahead
of the configuration build.

Live-verified on the rig image: crash now yields Exited (134) +
RestartCount climbing under `unless-stopped`, trace intact, with and
without init; full 8-node rig redeployed healthy with docker-init as PID 1.

Closes #34.

Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
2026-08-08 05:23:26 -04:00

213 lines
8.5 KiB
C#

using System.Reflection;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using ZB.MOM.WW.ScadaBridge.Host;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
[Collection(HostBootCollection.Name)]
public class HostStartupTests : IDisposable
{
private readonly List<IDisposable> _disposables = new();
public void Dispose()
{
foreach (var d in _disposables)
{
try { d.Dispose(); } catch { /* best effort */ }
}
}
[Fact]
public void CentralRole_StartsWithoutError()
{
// WebApplicationFactory replays Program.Main, which reads config from files.
// Set the environment to Central so appsettings.Central.json is loaded,
// and set DOTNET_ENVIRONMENT before the factory creates the host.
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
// Host-003: connection strings are externalised; supply them via env vars.
using var dbEnv = new CentralDbTestEnvironment();
try
{
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
var factory = new WebApplicationFactory<Program>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, config) =>
{
config.AddInMemoryCollection(new Dictionary<string, string?>
{
["ScadaBridge:Node:NodeName"] = "central-a",
["ScadaBridge:Node:NodeHostname"] = "localhost",
["ScadaBridge:Node:RemotingPort"] = "0",
["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
["ScadaBridge:Database:SkipMigrations"] = "true",
});
});
builder.UseSetting("ScadaBridge:Node:Role", "Central");
builder.UseSetting("ScadaBridge:Database:SkipMigrations", "true");
});
_disposables.Add(factory);
// Creating the server exercises the full DI container build and startup pipeline
var client = factory.CreateClient();
_disposables.Add(client);
// If we get here without exception, the central host started successfully
Assert.NotNull(client);
}
finally
{
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
}
}
[Fact]
public void SiteRole_StartsWithoutError()
{
var builder = WebApplication.CreateBuilder();
builder.Configuration.Sources.Clear();
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["ScadaBridge:Node:Role"] = "Site",
["ScadaBridge:Node:NodeName"] = "node-a",
["ScadaBridge:Node:NodeHostname"] = "test-site",
["ScadaBridge:Node:SiteId"] = "TestSite",
["ScadaBridge:Node:RemotingPort"] = "0",
["ScadaBridge:Node:GrpcPort"] = "0",
});
builder.Services.AddGrpc();
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
// Remove AkkaHostedService from running
AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);
var app = builder.Build();
_disposables.Add(app);
// Build succeeds = DI container is valid and all services resolve
Assert.NotNull(app);
Assert.NotNull(app.Services);
}
[Fact]
public void SiteRole_ConfiguresKestrelForGrpc()
{
var builder = WebApplication.CreateBuilder();
builder.Configuration.Sources.Clear();
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["ScadaBridge:Node:Role"] = "Site",
["ScadaBridge:Node:NodeName"] = "node-a",
["ScadaBridge:Node:NodeHostname"] = "test-site",
["ScadaBridge:Node:SiteId"] = "TestSite",
["ScadaBridge:Node:RemotingPort"] = "0",
["ScadaBridge:Node:GrpcPort"] = "0",
});
builder.WebHost.ConfigureKestrel(options =>
{
options.ListenAnyIP(0, listenOptions =>
{
listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2;
});
});
builder.Services.AddGrpc();
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
// Remove AkkaHostedService from running
AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);
var app = builder.Build();
// Verify Kestrel IS configured (site now hosts gRPC via WebApplicationBuilder)
var serverType = Type.GetType(
"Microsoft.AspNetCore.Hosting.Server.IServer, Microsoft.AspNetCore.Hosting.Server.Abstractions");
if (serverType != null)
{
var server = app.Services.GetService(serverType);
Assert.NotNull(server);
}
(app as IDisposable)?.Dispose();
}
[Fact]
public void HostProject_DoesNotUseConditionalCompilation()
{
var hostProjectDir = FindHostProjectDirectory();
Assert.NotNull(hostProjectDir);
var sourceFiles = Directory.GetFiles(hostProjectDir, "*.cs", SearchOption.TopDirectoryOnly);
Assert.NotEmpty(sourceFiles);
foreach (var file in sourceFiles)
{
var content = File.ReadAllText(file);
Assert.DoesNotContain("#if", content);
Assert.DoesNotContain("#ifdef", content);
Assert.DoesNotContain("#ifndef", content);
Assert.DoesNotContain("#elif", content);
Assert.DoesNotContain("#else", content);
Assert.DoesNotContain("#endif", content);
}
}
[Fact]
public void Program_RegistersUnhandledExceptionExitHandler()
{
// ScadaBridge#34: in a container this process is PID 1, and Linux ignores the
// SIGABRT the runtime's crash path raises against PID 1 — an unhandled boot
// exception printed its banner and then spun the main thread at 100% CPU with
// the container still `running`, so the restart policy never fired. Program.cs
// therefore registers an AppDomain.UnhandledException handler that exits via
// the exit() syscall (which PID 1 CAN perform) before anything can throw.
// Behavioural coverage needs a real crashed process (see the compose comment
// and the issue's live reproductions); this pins the handler's existence so a
// refactor cannot silently reopen the wedge.
var hostProjectDir = FindHostProjectDirectory();
Assert.NotNull(hostProjectDir);
var program = File.ReadAllText(Path.Combine(hostProjectDir!, "Program.cs"));
Assert.Contains("AppDomain.CurrentDomain.UnhandledException +=", program);
Assert.Contains("Environment.Exit(134)", program);
// The registration must precede the first statement that can throw — the
// configuration build is the earliest (appsettings.json is non-optional).
var handlerAt = program.IndexOf("AppDomain.CurrentDomain.UnhandledException +=", StringComparison.Ordinal);
var configAt = program.IndexOf("new ConfigurationBuilder()", StringComparison.Ordinal);
Assert.True(handlerAt >= 0 && configAt >= 0 && handlerAt < configAt,
"the UnhandledException exit handler must be registered before the configuration build");
}
private static string? FindHostProjectDirectory()
{
// Walk up from the test assembly location to find the src directory
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
var dir = new DirectoryInfo(assemblyDir);
while (dir != null)
{
var hostPath = Path.Combine(dir.FullName, "src", "ZB.MOM.WW.ScadaBridge.Host");
if (Directory.Exists(hostPath))
return hostPath;
dir = dir.Parent;
}
return null;
}
}