fix(historian): capture while the recording Primary is unreachable (#672)
A SIGKILLed Primary stays an unreachable-but-not-downed member until auto-down, and the redundancy singleton usually dies with it, so the standby's cached snapshot kept naming it Primary and nobody captured for ~30 s. DriverHostActor now holds its own ReachabilityEvent (+MemberUp) subscription, caches the last snapshot, and excludes unreachable peers from the historization peer-is-Primary predicate only, republishing just the capture verdict on a reachability change. Alarm drain and the Primary data-plane verdict are unchanged. Gated by ContinuousHistorization:CaptureWhilePrimaryUnreachable (default true). Tests: message-driven cases with real ReachabilityEvents (incl. a falsification control on the drain/data-plane verdicts) plus a real two-node cluster crash test with downing disabled.
This commit is contained in:
Joseph Doherty
+309
@@ -1,8 +1,13 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Reflection;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Util;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
@@ -134,8 +139,312 @@ public sealed class DriverHostActorHistorizationRoleViewTests : RuntimeActorTest
|
||||
AwaitAssert(() => view.Published.ShouldBe([false, true]), duration: Timeout);
|
||||
}
|
||||
|
||||
// ---------------- Gitea #672: an unreachable recording Primary ----------------
|
||||
//
|
||||
// A SIGKILLed Primary stays a member (unreachable, not downed) for the failure-detector + auto-down
|
||||
// window, and the redundancy singleton usually died with it — so the survivor's cached snapshot keeps
|
||||
// naming the dead node Primary and nobody captures. These drive REAL Akka ReachabilityEvents (built by
|
||||
// reflection, as the ControlPlane tests do) into the host, so the host:port projection is exercised too.
|
||||
|
||||
private static readonly NodeId Self672 = NodeId.Parse("self-node:4053");
|
||||
private static readonly NodeId Peer672 = NodeId.Parse("mesh-peer:4053");
|
||||
private static readonly NodeId Foreign672 = NodeId.Parse("other-cluster:4053");
|
||||
|
||||
/// <summary>
|
||||
/// An unreachable mesh-partner Primary opens the capture gate and its return closes it again — and
|
||||
/// neither moves the alarm-drain verdict nor the Primary data-plane verdict.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The drain/data-plane half is the falsification control: a lazier fix that excluded unreachable
|
||||
/// nodes from every peer question (or published the whole verdict set on a reachability change)
|
||||
/// would let a Secondary claim the Sparkplug host id / drain the replicated alarm queue during a
|
||||
/// mere failure-detector blip.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void An_unreachable_mesh_primary_opens_capture_and_its_return_closes_it_without_touching_other_verdicts()
|
||||
{
|
||||
var hist = new RecordingHistorizationView();
|
||||
var others = new RecordingOtherViews();
|
||||
IActorRef host = SpawnHost672(hist, others, captureWhilePrimaryUnreachable: true);
|
||||
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
|
||||
AwaitAssert(() => others.Drained.ShouldNotBeEmpty(), duration: Timeout);
|
||||
var drainedBefore = others.Drained;
|
||||
var primaryBefore = others.Primary;
|
||||
drainedBefore[^1].ShouldBeFalse(); // queue-sharing mesh peer is Primary ⇒ drain stands down
|
||||
primaryBefore[^1].ShouldBeFalse(); // Secondary ⇒ not the data-plane Primary
|
||||
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
|
||||
|
||||
// Same handler run that published `true` above — anything it published elsewhere is already recorded.
|
||||
others.Drained.ShouldBe(drainedBefore);
|
||||
others.Primary.ShouldBe(primaryBefore);
|
||||
others.Current.ShouldDrainAlarmHistory.ShouldBeFalse();
|
||||
others.Current.ShouldActAsPrimary.ShouldBeFalse();
|
||||
|
||||
host.Tell(new ClusterEvent.ReachableMember(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true, false]), duration: Timeout);
|
||||
others.Drained.ShouldBe(drainedBefore);
|
||||
others.Primary.ShouldBe(primaryBefore);
|
||||
}
|
||||
|
||||
/// <summary>A peer outside this node's mesh going unreachable changes nothing.</summary>
|
||||
[Fact]
|
||||
public void An_unreachable_out_of_scope_primary_changes_nothing()
|
||||
{
|
||||
var hist = new RecordingHistorizationView();
|
||||
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
|
||||
|
||||
// Both a foreign-cluster Primary (never stood down for) and the mesh Primary (stood down for).
|
||||
host.Tell(Snapshot(
|
||||
(Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary), (Foreign672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
|
||||
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Foreign672)));
|
||||
// Positive control: the mesh peer's loss, sent AFTER, must be the only change — had the foreign
|
||||
// one published anything it would sit between the two.
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
|
||||
}
|
||||
|
||||
/// <summary>This node's own reachability (as gossiped by others) is not a peer question.</summary>
|
||||
[Fact]
|
||||
public void Self_unreachable_is_ignored()
|
||||
{
|
||||
var hist = new RecordingHistorizationView();
|
||||
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
|
||||
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
|
||||
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Self672)));
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
|
||||
|
||||
// And self "returning" does not close what the peer's loss opened.
|
||||
host.Tell(new ClusterEvent.ReachableMember(MemberAt(Self672)));
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true, true]), duration: Timeout);
|
||||
}
|
||||
|
||||
/// <summary><c>CaptureWhilePrimaryUnreachable = false</c> restores the pre-#672 behaviour.</summary>
|
||||
[Fact]
|
||||
public void With_the_option_off_an_unreachable_primary_keeps_the_gate_closed()
|
||||
{
|
||||
var hist = new RecordingHistorizationView();
|
||||
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: false);
|
||||
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
|
||||
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
|
||||
// Positive control: a re-sent snapshot publishes on every arrival, so a `true` from the
|
||||
// reachability event would show up between the two `false`s.
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, false]), duration: Timeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A fresh snapshot naming this node Primary — the singleton re-homed after auto-down removed the
|
||||
/// dead peer — keeps the gate open; the removal itself must not flap it closed in between.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_snapshot_naming_me_primary_after_the_peer_is_removed_keeps_capturing()
|
||||
{
|
||||
var hist = new RecordingHistorizationView();
|
||||
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
|
||||
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
|
||||
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
|
||||
|
||||
host.Tell(new ClusterEvent.MemberRemoved(MemberAt(Peer672, MemberStatus.Removed), MemberStatus.Down));
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true, true]), duration: Timeout);
|
||||
}
|
||||
|
||||
/// <summary>A re-joined incarnation at the dead peer's address is reachable again.</summary>
|
||||
[Fact]
|
||||
public void A_rejoined_peer_is_reachable_again()
|
||||
{
|
||||
var hist = new RecordingHistorizationView();
|
||||
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
|
||||
|
||||
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
|
||||
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
|
||||
|
||||
host.Tell(new ClusterEvent.MemberUp(MemberAt(Peer672)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true, false]), duration: Timeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// End to end on a REAL two-node cluster: crash the recording Primary and the survivor's capture gate
|
||||
/// opens on failure detection alone — while the dead node is still a member, i.e. well before any
|
||||
/// auto-down could have removed it and re-homed the redundancy singleton.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Downing is disabled outright here, so the dead node stays a member forever: the only signal that
|
||||
/// can open the gate is the failure detector's <c>UnreachableMember</c>. The crash is
|
||||
/// <c>Terminate()</c> with <c>run-by-actor-system-terminate = off</c>, which gossips no Leave — a
|
||||
/// process death, not a graceful drain (pattern from <c>SelfFirstSeedBootstrapTests</c>).
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task A_crashed_primary_opens_the_survivors_capture_gate_before_it_is_downed()
|
||||
{
|
||||
var portA = FreePort();
|
||||
var portB = FreePort();
|
||||
const string systemName = "otopcua-672";
|
||||
var sysA = ActorSystem.Create(systemName, TwoNodeHocon(portA, portA));
|
||||
var sysB = ActorSystem.Create(systemName, TwoNodeHocon(portB, portA));
|
||||
try
|
||||
{
|
||||
var clusterB = Akka.Cluster.Cluster.Get(sysB);
|
||||
AwaitCondition(
|
||||
() => clusterB.State.Members.Count(m => m.Status == MemberStatus.Up) == 2,
|
||||
TimeSpan.FromSeconds(30));
|
||||
|
||||
var nodeA = NodeId.Parse($"127.0.0.1:{portA}");
|
||||
var nodeB = NodeId.Parse($"127.0.0.1:{portB}");
|
||||
var hist = new RecordingHistorizationView();
|
||||
IActorRef host = sysB.ActorOf(DriverHostActor.Props(
|
||||
dbFactory: null,
|
||||
nodeB,
|
||||
ackRouter: null,
|
||||
driverMemberCountProvider: () => 2,
|
||||
historizationRoleView: hist,
|
||||
sharesHistorizationScopeWithLocal: node => node == nodeA));
|
||||
|
||||
host.Tell(Snapshot((nodeB, RedundancyRole.Secondary), (nodeA, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
|
||||
|
||||
await sysA.Terminate();
|
||||
|
||||
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: TimeSpan.FromSeconds(30));
|
||||
// Still a member: the gate opened on reachability, not on a removal + fresh snapshot.
|
||||
clusterB.State.Members.ShouldContain(m => m.Address.Port == portA);
|
||||
clusterB.State.Unreachable.ShouldContain(m => m.Address.Port == portA);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await sysB.Terminate();
|
||||
if (!sysA.WhenTerminated.IsCompleted) await sysA.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private static int FreePort()
|
||||
{
|
||||
var listener = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
var port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port;
|
||||
listener.Stop();
|
||||
return port;
|
||||
}
|
||||
|
||||
private static Akka.Configuration.Config TwoNodeHocon(int selfPort, int seedPort) =>
|
||||
Akka.Configuration.ConfigurationFactory.ParseString($@"
|
||||
akka {{
|
||||
loglevel = ""WARNING""
|
||||
extensions = [""Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools""]
|
||||
actor.provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
|
||||
remote.dot-netty.tcp {{ hostname = ""127.0.0.1"", port = {selfPort} }}
|
||||
coordinated-shutdown.run-by-actor-system-terminate = off
|
||||
cluster {{
|
||||
seed-nodes = [""akka.tcp://otopcua-672@127.0.0.1:{seedPort}""]
|
||||
roles = [""driver""]
|
||||
downing-provider-class = """"
|
||||
run-coordinated-shutdown-when-down = off
|
||||
failure-detector {{ heartbeat-interval = 1s, acceptable-heartbeat-pause = 3s }}
|
||||
}}
|
||||
}}");
|
||||
|
||||
// ---------------- helpers ----------------
|
||||
|
||||
/// <summary>Records the drain and data-plane verdicts the host publishes.</summary>
|
||||
private sealed class RecordingOtherViews : IRedundancyRoleView, IDriverPrimaryRoleSource
|
||||
{
|
||||
private readonly List<bool> _drained = [];
|
||||
private readonly List<bool> _primary = [];
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
public RedundancyRoleView Current { get; } = new();
|
||||
|
||||
public IReadOnlyList<bool> Drained
|
||||
{
|
||||
get { lock (_gate) return _drained.ToArray(); }
|
||||
}
|
||||
|
||||
public IReadOnlyList<bool> Primary
|
||||
{
|
||||
get { lock (_gate) return _primary.ToArray(); }
|
||||
}
|
||||
|
||||
public bool ShouldDrainAlarmHistory => Current.ShouldDrainAlarmHistory;
|
||||
|
||||
public bool ShouldActAsPrimary => Current.ShouldActAsPrimary;
|
||||
|
||||
public void Publish(bool shouldDrainAlarmHistory)
|
||||
{
|
||||
lock (_gate) _drained.Add(shouldDrainAlarmHistory);
|
||||
Current.Publish(shouldDrainAlarmHistory);
|
||||
}
|
||||
|
||||
public void PublishPrimaryRole(bool shouldActAsPrimary)
|
||||
{
|
||||
lock (_gate) _primary.Add(shouldActAsPrimary);
|
||||
Current.PublishPrimaryRole(shouldActAsPrimary);
|
||||
}
|
||||
}
|
||||
|
||||
private IActorRef SpawnHost672(
|
||||
IHistorizationRoleView view, RecordingOtherViews? others, bool captureWhilePrimaryUnreachable) =>
|
||||
Sys.ActorOf(DriverHostActor.Props(
|
||||
NewInMemoryDbFactory(),
|
||||
Self672,
|
||||
ackRouter: null,
|
||||
driverMemberCountProvider: () => 2,
|
||||
redundancyRoleView: others,
|
||||
primaryRoleSource: others,
|
||||
historizationRoleView: view,
|
||||
sharesHistorizationScopeWithLocal: node => node == Peer672 || node == Self672,
|
||||
sharesAlarmQueueWithLocal: node => node == Peer672,
|
||||
captureWhilePrimaryUnreachable: captureWhilePrimaryUnreachable));
|
||||
|
||||
private static int _memberUid;
|
||||
|
||||
// Akka's Member.Create factories are internal — same reflection idiom as the ControlPlane tests.
|
||||
private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod(
|
||||
"Create",
|
||||
BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public,
|
||||
binder: null,
|
||||
types:
|
||||
[
|
||||
typeof(UniqueAddress), typeof(int), typeof(MemberStatus),
|
||||
typeof(ImmutableHashSet<string>), typeof(AppVersion),
|
||||
],
|
||||
modifiers: null)
|
||||
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found — Akka API changed");
|
||||
|
||||
private static Member MemberAt(NodeId node, MemberStatus status = MemberStatus.Up)
|
||||
{
|
||||
var text = node.ToString();
|
||||
var colon = text.LastIndexOf(':');
|
||||
return (Member)MemberCreate.Invoke(null,
|
||||
[
|
||||
new UniqueAddress(
|
||||
new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])),
|
||||
Interlocked.Increment(ref _memberUid)),
|
||||
1,
|
||||
status,
|
||||
ImmutableHashSet.Create("driver"),
|
||||
AppVersion.Zero,
|
||||
])!;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A view that remembers every published decision, so a test asserts on the actor's ACTION
|
||||
/// rather than on a state its open seed already satisfies.
|
||||
|
||||
+1
@@ -54,6 +54,7 @@ public sealed class ContinuousHistorizationOptionsTests
|
||||
Assert.Equal(8, options.DrainConcurrency);
|
||||
Assert.Equal(500_000, options.Capacity);
|
||||
Assert.Equal(5, options.MaxWriteAttempts);
|
||||
Assert.True(options.CaptureWhilePrimaryUnreachable); // Gitea #672
|
||||
}
|
||||
|
||||
/// <summary>A fully-configured enabled recorder produces no warnings.</summary>
|
||||
|
||||
Reference in New Issue
Block a user