952db02aab
A node frozen past failure detection + StableAfter, or both sides of a partition that heals after each downed the other, used to stay a second active node until an operator restarted one: the peer quarantines the removed UID and Akka.Cluster 1.5.62 never acts on ThisActorSystemQuarantinedEvent. ShutOutRestart (child of the LeaderAutoDown downing actor; opt-out via ScadaBridge:Cluster:LeaderAutoDown:RestartWhenShutOut, default on for leader-auto-down only) watches two signals: - S1 ThisActorSystemQuarantinedEvent: the peer quarantined this node. If the peer is still a live member here, only this side can be shut out -> restart now. - S2 AssociationErrorEvent 'has a UID that has been quarantined': the peer still runs the incarnation this node removed. A healed partition yields only S2, on both sides. Wait ShutOutGrace (60s) for a frozen peer to restart itself, then tiebreak. Tiebreak: the higher address (Akka Member.AddressOrdering) restarts, the lower stays and restarts only if still split one grace later. Removed peers are probed (Identify, 5s -> 60s back-off) since nothing else contacts them after mutual removal. Restart = Cluster.Down(self) -> existing self-down path. The Host now exits 75 after an unplanned ActorSystem termination (and sets it on the Windows service before it reports stopped): SCM recovery with failureflag 1 only restarts a service that stops with a non-zero code, and the watchdog used to exit 0. Shipped default strategy stays auto-down. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
105 lines
4.4 KiB
C#
105 lines
4.4 KiB
C#
using Akka.Actor;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using NSubstitute;
|
|
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
|
using ZB.MOM.WW.ScadaBridge.Communication;
|
|
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
|
|
|
/// <summary>
|
|
/// With SBR down-if-alone +
|
|
/// run-coordinated-shutdown-when-down live, a self-downed node terminates its
|
|
/// ActorSystem out from under the still-running Host process. Unless the process
|
|
/// then exits, the node idles forever — serving nothing, restarted by nobody.
|
|
/// The recovery contract: unexpected ActorSystem termination ⇒ StopApplication()
|
|
/// ⇒ the service supervisor restarts the process ⇒ it rejoins as a fresh
|
|
/// incarnation. A termination that IS the host's own StopAsync must NOT trip it.
|
|
/// </summary>
|
|
public sealed class UnexpectedTerminationTests
|
|
{
|
|
private static int FreePort()
|
|
{
|
|
var l = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
|
|
l.Start();
|
|
var p = ((System.Net.IPEndPoint)l.LocalEndpoint).Port;
|
|
l.Stop();
|
|
return p;
|
|
}
|
|
|
|
private static AkkaHostedService CreateAkkaHostedService(IHostApplicationLifetime lifetime)
|
|
{
|
|
var port = FreePort();
|
|
var self = $"akka.tcp://scadabridge@127.0.0.1:{port}";
|
|
return new AkkaHostedService(
|
|
new ServiceCollection().BuildServiceProvider(),
|
|
Options.Create(new NodeOptions { Role = "Site", NodeHostname = "127.0.0.1", RemotingPort = port, SiteId = "site-a" }),
|
|
Options.Create(new ClusterOptions
|
|
{
|
|
SeedNodes = new List<string> { self },
|
|
AllowSingleNodeCluster = true,
|
|
SplitBrainResolverStrategy = "keep-oldest",
|
|
StableAfter = TimeSpan.FromSeconds(3),
|
|
HeartbeatInterval = TimeSpan.FromMilliseconds(500),
|
|
FailureDetectionThreshold = TimeSpan.FromSeconds(2),
|
|
MinNrOfMembers = 1,
|
|
DownIfAlone = true,
|
|
}),
|
|
Options.Create(new CommunicationOptions()),
|
|
NullLogger<AkkaHostedService>.Instance,
|
|
lifetime);
|
|
}
|
|
|
|
/// <summary>Verifies that the host application is stopped when the actor system terminates outside of a stop request.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ActorSystemTerminatedOutsideStopAsync_StopsHostApplication()
|
|
{
|
|
var lifetime = Substitute.For<IHostApplicationLifetime>();
|
|
var service = CreateAkkaHostedService(lifetime);
|
|
// Create the system (StartAsync registers actors; the system is created
|
|
// by GetOrCreateActorSystem, which wires the WhenTerminated watchdog).
|
|
service.GetOrCreateActorSystem();
|
|
var system = service.ActorSystem!;
|
|
|
|
await system.Terminate(); // simulated self-down, NOT StopAsync
|
|
await Task.Delay(500);
|
|
|
|
lifetime.Received(1).StopApplication();
|
|
// ScadaBridge#132: Program turns this flag into the non-zero SelfDownExitCode.
|
|
Assert.True(service.TerminatedOutsideHostShutdown);
|
|
}
|
|
|
|
/// <summary>Verifies that a normal stop does not trigger an application stop.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task StopAsync_DoesNotTriggerStopApplication()
|
|
{
|
|
var lifetime = Substitute.For<IHostApplicationLifetime>();
|
|
var service = CreateAkkaHostedService(lifetime);
|
|
service.GetOrCreateActorSystem();
|
|
|
|
await service.StopAsync(CancellationToken.None);
|
|
await Task.Delay(500);
|
|
|
|
lifetime.DidNotReceive().StopApplication();
|
|
Assert.False(service.TerminatedOutsideHostShutdown);
|
|
}
|
|
|
|
/// <summary>
|
|
/// ScadaBridge#132: the self-down exit code is non-zero (Windows service recovery with
|
|
/// failureflag 1 restarts only on a non-zero exit) and is only pushed into a Windows service
|
|
/// lifetime — a console / docker lifetime is left alone.
|
|
/// </summary>
|
|
[Fact]
|
|
public void SelfDownExitCode_IsNonZero_AndOnlyAppliedToAServiceLifetime()
|
|
{
|
|
Assert.NotEqual(0, AkkaHostedService.SelfDownExitCode);
|
|
Assert.False(AkkaHostedService.MarkServiceSelfDownExit(null));
|
|
Assert.False(AkkaHostedService.MarkServiceSelfDownExit(Substitute.For<IHostLifetime>()));
|
|
}
|
|
}
|