e9c412e528
Root cause: dotnet runs as container PID 1 and Linux ignores default-action signals sent to PID 1, so the runtime's unhandled-exception path (banner, then abort() -> SIGABRT) could never terminate the process — it printed the trace and spun the main thread at 100% CPU with the container `running`, so `restart: unless-stopped` never fired. Reproduced deterministically: same StartupValidator throw exits 134 under an init process and wedges without one. Two layers, each covering the other's gap: - Program.cs registers an AppDomain.UnhandledException handler before the first statement that can throw: prints the trace, best-effort flushes Serilog, Environment.Exit(134) — exit() is a syscall PID 1 CAN perform, 134 preserves the 128+SIGABRT crash code, and it covers every thread, not just the boot window. It cannot fire under WebApplicationFactory (the test host catches entry-point exceptions), so the designed boot-refusal exceptions still propagate to tests unchanged. - docker-compose: init: true on all 8 nodes for the crash paths that bypass the managed event (Environment.FailFast, runtime-internal aborts). The CoordinatedShutdown no-Environment.Exit guard gains a precise carve-out (exactly one call, only inside the handler); Environment.Exit still fires the CLR shutdown hook Akka binds via run-by-clr-shutdown-hook = on, so the crash path skips nothing abort() kept. New pin test keeps the handler ahead of the configuration build. Live-verified on the rig image: crash now yields Exited (134) + RestartCount climbing under `unless-stopped`, trace intact, with and without init; full 8-node rig redeployed healthy with docker-init as PID 1. Closes #34. Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
139 lines
6.0 KiB
C#
139 lines
6.0 KiB
C#
using System.Reflection;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
|
|
|
/// <summary>
|
|
/// WP-16: Tests for CoordinatedShutdown configuration.
|
|
/// Verifies no Environment.Exit calls exist in source and HOCON config is correct.
|
|
/// </summary>
|
|
public class CoordinatedShutdownTests
|
|
{
|
|
[Fact]
|
|
public void HostSource_DoesNotContainEnvironmentExit()
|
|
{
|
|
var hostProjectDir = FindHostProjectDirectory();
|
|
Assert.NotNull(hostProjectDir);
|
|
|
|
var sourceFiles = Directory.GetFiles(hostProjectDir, "*.cs", SearchOption.AllDirectories);
|
|
Assert.NotEmpty(sourceFiles);
|
|
|
|
foreach (var file in sourceFiles)
|
|
{
|
|
var content = File.ReadAllText(file);
|
|
|
|
// Sole permitted call site: Program.cs's AppDomain.UnhandledException handler
|
|
// (ScadaBridge#34). That is the crash path — the alternative there was never a
|
|
// CoordinatedShutdown but the runtime's abort(), which cannot terminate PID 1
|
|
// and wedged the container; Environment.Exit still fires the CLR shutdown hook
|
|
// Akka binds via run-by-clr-shutdown-hook = on, so it skips nothing abort kept.
|
|
// Everywhere else the original rule stands: no code path may bypass
|
|
// CoordinatedShutdown by exiting directly.
|
|
if (Path.GetFileName(file) == "Program.cs")
|
|
{
|
|
var occurrences = CountOccurrences(content, "Environment.Exit(");
|
|
Assert.Equal(1, occurrences);
|
|
|
|
var handlerAt = content.IndexOf(
|
|
"AppDomain.CurrentDomain.UnhandledException +=", StringComparison.Ordinal);
|
|
Assert.True(handlerAt >= 0, "Program.cs must register the UnhandledException handler");
|
|
var handlerEnd = content.IndexOf("};", handlerAt, StringComparison.Ordinal);
|
|
var exitAt = content.IndexOf("Environment.Exit(", StringComparison.Ordinal);
|
|
Assert.True(exitAt > handlerAt && exitAt < handlerEnd,
|
|
"Environment.Exit in Program.cs is only permitted inside the UnhandledException handler");
|
|
continue;
|
|
}
|
|
|
|
Assert.DoesNotContain("Environment.Exit", content,
|
|
StringComparison.Ordinal);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void AkkaHostedService_HoconConfig_IncludesCoordinatedShutdownSettings()
|
|
{
|
|
// Read the AkkaHostedService source to verify HOCON configuration
|
|
var hostProjectDir = FindHostProjectDirectory();
|
|
Assert.NotNull(hostProjectDir);
|
|
|
|
var akkaServiceFile = Path.Combine(hostProjectDir, "Actors", "AkkaHostedService.cs");
|
|
Assert.True(File.Exists(akkaServiceFile), $"AkkaHostedService.cs not found at {akkaServiceFile}");
|
|
|
|
var content = File.ReadAllText(akkaServiceFile);
|
|
|
|
// Verify critical HOCON settings are present
|
|
Assert.Contains("run-by-clr-shutdown-hook = on", content);
|
|
Assert.Contains("run-coordinated-shutdown-when-down = on", content);
|
|
}
|
|
|
|
[Fact]
|
|
public void AllCentralSingletons_RegisterThroughRegistrarWithDrain()
|
|
{
|
|
var hostProjectDir = FindHostProjectDirectory();
|
|
Assert.NotNull(hostProjectDir);
|
|
var content = File.ReadAllText(Path.Combine(hostProjectDir!, "Actors", "AkkaHostedService.cs"));
|
|
|
|
// Review 01 [Medium]: notification-outbox and audit-log-ingest were the
|
|
// only central singletons WITHOUT a cluster-leave drain task. All seven
|
|
// now go through SingletonRegistrar.Start, which always adds the
|
|
// PhaseClusterLeave GracefulStop drain.
|
|
Assert.Contains("SingletonRegistrar.Start(", content);
|
|
foreach (var name in new[] { "notification-outbox", "audit-log-ingest", "site-call-audit",
|
|
"audit-log-purge", "site-audit-reconciliation",
|
|
"kpi-history-recorder", "pending-deployment-purge" })
|
|
{
|
|
Assert.Contains($"\"{name}\"", content);
|
|
}
|
|
|
|
// No hand-rolled ClusterSingletonManager registrations remain anywhere:
|
|
// the two role-scoped SITE singletons now also go through the registrar
|
|
// (SingletonRegistrar.Start(..., role: siteRole)), so every singleton in
|
|
// the file is created through the registrar with a drain task.
|
|
Assert.Equal(0, CountOccurrences(content, "ClusterSingletonManager.Props("));
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteSingletons_RegisterThroughRegistrarWithDrain()
|
|
{
|
|
var hostProjectDir = FindHostProjectDirectory();
|
|
Assert.NotNull(hostProjectDir);
|
|
var content = File.ReadAllText(Path.Combine(hostProjectDir!, "Actors", "AkkaHostedService.cs"));
|
|
|
|
// Round-2 N5: the two role-scoped SITE singletons previously stayed
|
|
// hand-rolled (bare PoisonPill, no PhaseClusterLeave drain). They now
|
|
// go through SingletonRegistrar.Start(..., role: siteRole), which
|
|
// always adds the GracefulStop drain — in-flight SQLite writes
|
|
// (static overrides, native_alarm_state) complete before handover.
|
|
Assert.Contains("\"deployment-manager\"", content);
|
|
Assert.Contains("\"event-log-handler\"", content);
|
|
Assert.Contains("role: siteRole", content);
|
|
Assert.Equal(0, CountOccurrences(content, "ClusterSingletonManager.Props("));
|
|
}
|
|
|
|
private static int CountOccurrences(string haystack, string needle)
|
|
{
|
|
int count = 0, i = 0;
|
|
while ((i = haystack.IndexOf(needle, i, StringComparison.Ordinal)) >= 0)
|
|
{
|
|
count++;
|
|
i += needle.Length;
|
|
}
|
|
return count;
|
|
}
|
|
|
|
private static string? FindHostProjectDirectory()
|
|
{
|
|
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
|
|
var dir = new DirectoryInfo(assemblyDir);
|
|
|
|
while (dir != null)
|
|
{
|
|
var hostPath = Path.Combine(dir.FullName, "src", "ZB.MOM.WW.ScadaBridge.Host");
|
|
if (Directory.Exists(hostPath))
|
|
return hostPath;
|
|
dir = dir.Parent;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|