Joseph Doherty
b0eb653bad
Corrects the #459 finding. 2-node keep-oldest recovery works fine (the ScadaBridge sister project proves it); OtOpcUa was missing the supervision pieces that make it automatic, and docs/Redundancy.md wrongly claimed in-place oldest-crash failover. Mechanism (confirmed on a 2-container rig + by decompiling Akka KeepOldest.OldestDecision): on an OLDEST-node crash keep-oldest downs the LONE survivor (DownReachable including myself) — down-if-alone can't rescue a lone survivor (its branch needs >=2 survivors). Recovery is exit-and-rejoin: run-coordinated-shutdown-when-down terminates the node and the service supervisor restarts it. My earlier 'total outage' was a docker-dev artifact (no restart policy); production Install-Services.ps1 already has sc.exe failure restart. Changes (ScadaBridge parity): - ActorSystemTerminationWatchdog (Host, registered after AddAkka): watches ActorSystem.WhenTerminated and on an unexpected self-down calls StopApplication so the process exits (supervisor restarts it) instead of idling with a dead actor system. Distinguishes graceful shutdown via _stopRequested + ApplicationStopping. 3 unit tests. - docker-dev: restart: unless-stopped on the host anchor (models production supervision) + both redundancy peers in SeedNodes so a restarted node re-forms via either peer. - docs/Redundancy.md: rewrote the split-brain recovery section — younger-loss = in-place fast failover; oldest-loss = exit-and-rejoin under supervision (not in-place); the three requirements (supervisor + watchdog + both-node seeds); flagged HardKillFailoverTests as non-representative (Transport.Shutdown, not a real crash). Instant in-place takeover on ANY single loss needs 3+ members. Cluster.Tests 29/29 (SBR guards), watchdog tests 3/3, full solution builds. Live re-verify of the watchdog image pending (host docker disk full).
93 lines
4.2 KiB
C#
93 lines
4.2 KiB
C#
using Akka.Actor;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Host;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// Unit coverage for the #459 down-if-alone recovery watchdog: an unexpected
|
|
/// <see cref="ActorSystem"/> termination (SBR self-down) must stop the host so the supervisor
|
|
/// restarts it, while a graceful host shutdown must stay silent and not re-trigger stop.
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class ActorSystemTerminationWatchdogTests
|
|
{
|
|
/// <summary>An unexpected ActorSystem termination stops the host so the supervisor restarts the node.</summary>
|
|
[Fact]
|
|
public async Task Unexpected_termination_stops_the_application()
|
|
{
|
|
var system = ActorSystem.Create("watchdog-unexpected");
|
|
var lifetime = new FakeHostApplicationLifetime();
|
|
var watchdog = new ActorSystemTerminationWatchdog(
|
|
() => system, lifetime, NullLogger<ActorSystemTerminationWatchdog>.Instance);
|
|
|
|
await watchdog.StartAsync(CancellationToken.None);
|
|
|
|
// Simulate the SBR self-down: the ActorSystem terminates while the host is still running.
|
|
await system.Terminate();
|
|
|
|
await WaitForAsync(() => lifetime.StopApplicationCalls > 0, TimeSpan.FromSeconds(5));
|
|
lifetime.StopApplicationCalls.ShouldBe(1, "an out-of-band ActorSystem termination must stop the host once");
|
|
}
|
|
|
|
/// <summary>A graceful host stop (StopAsync ran) leaves the termination continuation silent.</summary>
|
|
[Fact]
|
|
public async Task Graceful_stop_does_not_stop_the_application()
|
|
{
|
|
var system = ActorSystem.Create("watchdog-graceful");
|
|
var lifetime = new FakeHostApplicationLifetime();
|
|
var watchdog = new ActorSystemTerminationWatchdog(
|
|
() => system, lifetime, NullLogger<ActorSystemTerminationWatchdog>.Instance);
|
|
|
|
await watchdog.StartAsync(CancellationToken.None);
|
|
|
|
// Host is shutting down normally: StopAsync runs, THEN the system terminates.
|
|
await watchdog.StopAsync(CancellationToken.None);
|
|
await system.Terminate();
|
|
|
|
// Give any continuation a chance to run, then assert it stayed silent.
|
|
await Task.Delay(500, TestContext.Current.CancellationToken);
|
|
lifetime.StopApplicationCalls.ShouldBe(0, "a graceful shutdown must not re-trigger StopApplication");
|
|
}
|
|
|
|
/// <summary>Termination while the host is already stopping (ApplicationStopping fired) stays silent.</summary>
|
|
[Fact]
|
|
public async Task Termination_while_already_stopping_stays_silent()
|
|
{
|
|
var system = ActorSystem.Create("watchdog-already-stopping");
|
|
var lifetime = new FakeHostApplicationLifetime();
|
|
var watchdog = new ActorSystemTerminationWatchdog(
|
|
() => system, lifetime, NullLogger<ActorSystemTerminationWatchdog>.Instance);
|
|
|
|
await watchdog.StartAsync(CancellationToken.None);
|
|
|
|
lifetime.TriggerStopping(); // the host is already tearing down (ApplicationStopping cancelled)
|
|
await system.Terminate();
|
|
|
|
await Task.Delay(500, TestContext.Current.CancellationToken);
|
|
lifetime.StopApplicationCalls.ShouldBe(0, "termination during an in-progress host shutdown must stay silent");
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (!condition() && DateTime.UtcNow < deadline)
|
|
await Task.Delay(20);
|
|
}
|
|
|
|
/// <summary>Minimal <see cref="IHostApplicationLifetime"/> that records <see cref="StopApplication"/> calls.</summary>
|
|
private sealed class FakeHostApplicationLifetime : IHostApplicationLifetime
|
|
{
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
public int StopApplicationCalls { get; private set; }
|
|
public CancellationToken ApplicationStarted => CancellationToken.None;
|
|
public CancellationToken ApplicationStopping => _stopping.Token;
|
|
public CancellationToken ApplicationStopped => CancellationToken.None;
|
|
public void StopApplication() => StopApplicationCalls++;
|
|
public void TriggerStopping() => _stopping.Cancel();
|
|
}
|
|
}
|