using System.Collections.Immutable;
using System.Reflection;
using Akka.Actor;
using Akka.Cluster;
using Akka.Util;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
///
/// Gitea #537 — the bridge that carries the continuous-historization capture verdict out of
/// the actor and into , where the recorder reads it per value.
///
///
///
/// Both nodes of a redundant pair serve the same address space and see the same dependency-mux
/// fan-out, so an ungated recorder writes every sample twice. Measured on the docker-dev central
/// pair against a real gateway: the write counter advanced twice per value, and halved when one
/// node was stopped.
///
///
/// These cases share the role rule with but differ on
/// which peer counts. The alarm queue replicates, so the relevant peer is whoever holds a copy
/// of this node's rows; historization is fleet-wide by historian tag name, so the relevant peer
/// is whoever records this node's refs — a partner in its own per-cluster mesh.
///
///
public sealed class DriverHostActorHistorizationRoleViewTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-histview-test");
private static readonly NodeId MeshPeer = NodeId.Parse("mesh-peer:4053");
private static readonly NodeId ForeignNode = NodeId.Parse("other-cluster:4053");
private static readonly TimeSpan Timeout = PresenceBudget;
///
/// Before any snapshot arrives the view must read open, not closed.
///
///
/// A deployment that runs no redundancy never publishes here at all. Defaulting closed would
/// leave it with no trend data whatsoever and nothing to say why.
///
[Fact]
public void An_unpublished_view_captures()
=> new RedundancyRoleView().ShouldRecordHistorizedValues.ShouldBeTrue();
/// A Secondary stands down for a Primary in its own mesh — the duplicate-write case.
[Fact]
public void Secondary_stands_down_for_a_primary_in_its_own_mesh()
{
var view = new RecordingHistorizationView();
SpawnHost(view, sharesScopeWith: MeshPeer).Tell(Snapshot(
(TestNode, RedundancyRole.Secondary), (MeshPeer, RedundancyRole.Primary)));
AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout);
}
///
/// A Secondary whose Primary is in ANOTHER cluster keeps capturing.
///
///
/// That node runs a different configuration and records none of these tags, so standing down for
/// it would end this cluster's historization silently — the drain would simply report success on
/// an empty outbox forever. Same scope mismatch that once suspended the alarm drain fleet-wide.
///
[Fact]
public void Secondary_keeps_capturing_when_the_primary_records_other_tags()
{
var view = new RecordingHistorizationView();
SpawnHost(view, sharesScopeWith: MeshPeer).Tell(Snapshot(
(TestNode, RedundancyRole.Secondary), (ForeignNode, RedundancyRole.Primary)));
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
///
/// With NO way to establish which peers share this node's address space, it never stands down.
///
///
/// The predicate is null on a node carrying no cluster-scoped role — the legacy, pre-Phase-6
/// shape, where the redundancy election is fleet-wide and the elected Primary usually belongs to
/// some other cluster. Duplicates are recoverable; a silent gap is not, so unproven means
/// capture. Delete the null check in the wiring and this is the case that goes red.
///
[Fact]
public void An_unprovable_scope_never_stands_down()
{
var view = new RecordingHistorizationView();
SpawnHost(view, sharesScopeWith: null).Tell(Snapshot(
(TestNode, RedundancyRole.Secondary), (MeshPeer, RedundancyRole.Primary)));
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
/// Primary captures.
[Fact]
public void Primary_captures()
{
var view = new RecordingHistorizationView();
SpawnHost(view, sharesScopeWith: MeshPeer).Tell(Snapshot(
(TestNode, RedundancyRole.Primary), (MeshPeer, RedundancyRole.Secondary)));
// Published, not current — `true` is also the seed, so asserting the property would pass before
// the actor had processed anything.
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
///
/// Unknown role captures — the snapshot never names this node, the documented identity-mismatch
/// shape.
///
[Fact]
public void Unknown_role_captures()
{
var view = new RecordingHistorizationView();
SpawnHost(view, sharesScopeWith: MeshPeer).Tell(Snapshot(
(MeshPeer, RedundancyRole.Primary), (ForeignNode, RedundancyRole.Secondary)));
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
/// A promotion must MOVE the view, not merely leave it at its open seed.
[Fact]
public void A_promotion_reopens_a_closed_view()
{
var view = new RecordingHistorizationView();
IActorRef host = SpawnHost(view, sharesScopeWith: MeshPeer);
host.Tell(Snapshot((TestNode, RedundancyRole.Secondary), (MeshPeer, RedundancyRole.Primary)));
AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout);
host.Tell(Snapshot((TestNode, RedundancyRole.Primary), (MeshPeer, RedundancyRole.Secondary)));
AwaitAssert(() => view.Published.ShouldBe([false, true]), duration: Timeout);
}
// ---------------- Gitea #672: an unreachable recording Primary ----------------
//
// A SIGKILLed Primary stays a member (unreachable, not downed) for the failure-detector + auto-down
// window, and the redundancy singleton usually died with it — so the survivor's cached snapshot keeps
// naming the dead node Primary and nobody captures. These drive REAL Akka ReachabilityEvents (built by
// reflection, as the ControlPlane tests do) into the host, so the host:port projection is exercised too.
private static readonly NodeId Self672 = NodeId.Parse("self-node:4053");
private static readonly NodeId Peer672 = NodeId.Parse("mesh-peer:4053");
private static readonly NodeId Foreign672 = NodeId.Parse("other-cluster:4053");
///
/// An unreachable mesh-partner Primary opens the capture gate and its return closes it again — and
/// neither moves the alarm-drain verdict nor the Primary data-plane verdict.
///
///
/// The drain/data-plane half is the falsification control: a lazier fix that excluded unreachable
/// nodes from every peer question (or published the whole verdict set on a reachability change)
/// would let a Secondary claim the Sparkplug host id / drain the replicated alarm queue during a
/// mere failure-detector blip.
///
[Fact]
public void An_unreachable_mesh_primary_opens_capture_and_its_return_closes_it_without_touching_other_verdicts()
{
var hist = new RecordingHistorizationView();
var others = new RecordingOtherViews();
IActorRef host = SpawnHost672(hist, others, captureWhilePrimaryUnreachable: true);
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
AwaitAssert(() => others.Drained.ShouldNotBeEmpty(), duration: Timeout);
var drainedBefore = others.Drained;
var primaryBefore = others.Primary;
drainedBefore[^1].ShouldBeFalse(); // queue-sharing mesh peer is Primary ⇒ drain stands down
primaryBefore[^1].ShouldBeFalse(); // Secondary ⇒ not the data-plane Primary
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
// Same handler run that published `true` above — anything it published elsewhere is already recorded.
others.Drained.ShouldBe(drainedBefore);
others.Primary.ShouldBe(primaryBefore);
others.Current.ShouldDrainAlarmHistory.ShouldBeFalse();
others.Current.ShouldActAsPrimary.ShouldBeFalse();
host.Tell(new ClusterEvent.ReachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true, false]), duration: Timeout);
others.Drained.ShouldBe(drainedBefore);
others.Primary.ShouldBe(primaryBefore);
}
/// A peer outside this node's mesh going unreachable changes nothing.
[Fact]
public void An_unreachable_out_of_scope_primary_changes_nothing()
{
var hist = new RecordingHistorizationView();
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
// Both a foreign-cluster Primary (never stood down for) and the mesh Primary (stood down for).
host.Tell(Snapshot(
(Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary), (Foreign672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Foreign672)));
// Positive control: the mesh peer's loss, sent AFTER, must be the only change — had the foreign
// one published anything it would sit between the two.
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
}
/// This node's own reachability (as gossiped by others) is not a peer question.
[Fact]
public void Self_unreachable_is_ignored()
{
var hist = new RecordingHistorizationView();
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Self672)));
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
// Self "returning" is ignored too, and it does not close what the peer's loss opened. Events are
// processed in order, so the peer's return (which MUST close the gate) is the only thing that may
// appear next: had the self event published anything it would sit between `true` and `false`.
host.Tell(new ClusterEvent.ReachableMember(MemberAt(Self672)));
host.Tell(new ClusterEvent.ReachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true, false]), duration: Timeout);
}
///
/// A host that starts while its peer is ALREADY unreachable (the InitialStateAsEvents replay
/// delivers the UnreachableMember before the first redundancy snapshot) must still treat that
/// peer as unreachable when the snapshot naming it Primary finally arrives.
///
[Fact]
public void A_peer_unreachable_before_the_first_snapshot_still_opens_capture()
{
var hist = new RecordingHistorizationView();
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
// Exactly one publication, and it is `true`: no spurious `false` from treating the peer as reachable.
AwaitAssert(() => hist.Published.ShouldBe([true]), duration: Timeout);
}
/// CaptureWhilePrimaryUnreachable = false restores the pre-#672 behaviour.
[Fact]
public void With_the_option_off_an_unreachable_primary_keeps_the_gate_closed()
{
var hist = new RecordingHistorizationView();
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: false);
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
// Positive control: a re-sent snapshot publishes on every arrival, so a `true` from the
// reachability event would show up between the two `false`s.
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false, false]), duration: Timeout);
}
///
/// A fresh snapshot naming this node Primary — the singleton re-homed after auto-down removed the
/// dead peer — keeps the gate open; the removal itself must not flap it closed in between.
///
[Fact]
public void A_snapshot_naming_me_primary_after_the_peer_is_removed_keeps_capturing()
{
var hist = new RecordingHistorizationView();
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
host.Tell(new ClusterEvent.MemberRemoved(MemberAt(Peer672, MemberStatus.Removed), MemberStatus.Down));
host.Tell(Snapshot((Self672, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false, true, true]), duration: Timeout);
}
/// A re-joined incarnation at the dead peer's address is reachable again.
[Fact]
public void A_rejoined_peer_is_reachable_again()
{
var hist = new RecordingHistorizationView();
IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true);
host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary)));
host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout);
host.Tell(new ClusterEvent.MemberUp(MemberAt(Peer672)));
AwaitAssert(() => hist.Published.ShouldBe([false, true, false]), duration: Timeout);
}
///
/// End to end on a REAL two-node cluster: crash the recording Primary and the survivor's capture gate
/// opens on failure detection alone — while the dead node is still a member, i.e. well before any
/// auto-down could have removed it and re-homed the redundancy singleton.
///
///
/// Downing is disabled outright here, so the dead node stays a member forever: the only signal that
/// can open the gate is the failure detector's UnreachableMember. The crash is
/// Terminate() with run-by-actor-system-terminate = off, which gossips no Leave — a
/// process death, not a graceful drain (pattern from SelfFirstSeedBootstrapTests).
///
[Fact]
public async Task A_crashed_primary_opens_the_survivors_capture_gate_before_it_is_downed()
{
var portA = FreePort();
var portB = FreePort();
const string systemName = "otopcua-672";
var sysA = ActorSystem.Create(systemName, TwoNodeHocon(portA, portA));
var sysB = ActorSystem.Create(systemName, TwoNodeHocon(portB, portA));
try
{
var clusterB = Akka.Cluster.Cluster.Get(sysB);
AwaitCondition(
() => clusterB.State.Members.Count(m => m.Status == MemberStatus.Up) == 2,
TimeSpan.FromSeconds(30));
var nodeA = NodeId.Parse($"127.0.0.1:{portA}");
var nodeB = NodeId.Parse($"127.0.0.1:{portB}");
var hist = new RecordingHistorizationView();
IActorRef host = sysB.ActorOf(DriverHostActor.Props(
dbFactory: null,
nodeB,
ackRouter: null,
driverMemberCountProvider: () => 2,
historizationRoleView: hist,
sharesHistorizationScopeWithLocal: node => node == nodeA));
host.Tell(Snapshot((nodeB, RedundancyRole.Secondary), (nodeA, RedundancyRole.Primary)));
AwaitAssert(() => hist.Published.ShouldBe([false]), duration: Timeout);
await sysA.Terminate();
AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: TimeSpan.FromSeconds(30));
// Still a member: the gate opened on reachability, not on a removal + fresh snapshot.
clusterB.State.Members.ShouldContain(m => m.Address.Port == portA);
clusterB.State.Unreachable.ShouldContain(m => m.Address.Port == portA);
}
finally
{
await sysB.Terminate();
if (!sysA.WhenTerminated.IsCompleted) await sysA.Terminate();
}
}
private static int FreePort()
{
var listener = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
listener.Start();
var port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
return port;
}
private static Akka.Configuration.Config TwoNodeHocon(int selfPort, int seedPort) =>
Akka.Configuration.ConfigurationFactory.ParseString($@"
akka {{
loglevel = ""WARNING""
extensions = [""Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools""]
actor.provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
remote.dot-netty.tcp {{ hostname = ""127.0.0.1"", port = {selfPort} }}
coordinated-shutdown.run-by-actor-system-terminate = off
cluster {{
seed-nodes = [""akka.tcp://otopcua-672@127.0.0.1:{seedPort}""]
roles = [""driver""]
downing-provider-class = """"
run-coordinated-shutdown-when-down = off
failure-detector {{ heartbeat-interval = 1s, acceptable-heartbeat-pause = 3s }}
}}
}}");
// ---------------- helpers ----------------
/// Records the drain and data-plane verdicts the host publishes.
private sealed class RecordingOtherViews : IRedundancyRoleView, IDriverPrimaryRoleSource
{
private readonly List _drained = [];
private readonly List _primary = [];
private readonly Lock _gate = new();
public RedundancyRoleView Current { get; } = new();
public IReadOnlyList Drained
{
get { lock (_gate) return _drained.ToArray(); }
}
public IReadOnlyList Primary
{
get { lock (_gate) return _primary.ToArray(); }
}
public bool ShouldDrainAlarmHistory => Current.ShouldDrainAlarmHistory;
public bool ShouldActAsPrimary => Current.ShouldActAsPrimary;
public void Publish(bool shouldDrainAlarmHistory)
{
lock (_gate) _drained.Add(shouldDrainAlarmHistory);
Current.Publish(shouldDrainAlarmHistory);
}
public void PublishPrimaryRole(bool shouldActAsPrimary)
{
lock (_gate) _primary.Add(shouldActAsPrimary);
Current.PublishPrimaryRole(shouldActAsPrimary);
}
}
private IActorRef SpawnHost672(
IHistorizationRoleView view, RecordingOtherViews? others, bool captureWhilePrimaryUnreachable) =>
Sys.ActorOf(DriverHostActor.Props(
NewInMemoryDbFactory(),
Self672,
ackRouter: null,
driverMemberCountProvider: () => 2,
redundancyRoleView: others,
primaryRoleSource: others,
historizationRoleView: view,
sharesHistorizationScopeWithLocal: node => node == Peer672 || node == Self672,
sharesAlarmQueueWithLocal: node => node == Peer672,
captureWhilePrimaryUnreachable: captureWhilePrimaryUnreachable));
private static int _memberUid;
// Akka's Member.Create factories are internal — same reflection idiom as the ControlPlane tests.
private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod(
"Create",
BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public,
binder: null,
types:
[
typeof(UniqueAddress), typeof(int), typeof(MemberStatus),
typeof(ImmutableHashSet), typeof(AppVersion),
],
modifiers: null)
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found — Akka API changed");
private static Member MemberAt(NodeId node, MemberStatus status = MemberStatus.Up)
{
var text = node.ToString();
var colon = text.LastIndexOf(':');
return (Member)MemberCreate.Invoke(null,
[
new UniqueAddress(
new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])),
Interlocked.Increment(ref _memberUid)),
1,
status,
ImmutableHashSet.Create("driver"),
AppVersion.Zero,
])!;
}
///
/// A view that remembers every published decision, so a test asserts on the actor's ACTION
/// rather than on a state its open seed already satisfies.
///
private sealed class RecordingHistorizationView : IHistorizationRoleView
{
private readonly List _published = [];
private readonly Lock _gate = new();
private readonly RedundancyRoleView _inner = new();
/// Every value published so far, in order.
public IReadOnlyList Published
{
get { lock (_gate) return _published.ToArray(); }
}
public bool ShouldRecordHistorizedValues => _inner.ShouldRecordHistorizedValues;
public void PublishHistorizationRole(bool shouldRecordHistorizedValues)
{
lock (_gate) _published.Add(shouldRecordHistorizedValues);
_inner.PublishHistorizationRole(shouldRecordHistorizedValues);
}
}
private IActorRef SpawnHost(IHistorizationRoleView view, NodeId? sharesScopeWith) =>
Sys.ActorOf(DriverHostActor.Props(
NewInMemoryDbFactory(),
TestNode,
ackRouter: null,
driverMemberCountProvider: () => 2,
historizationRoleView: view,
sharesHistorizationScopeWithLocal:
sharesScopeWith is { } peer ? node => node == peer : null));
private static RedundancyStateChanged Snapshot(params (NodeId Node, RedundancyRole Role)[] nodes) =>
new(
nodes.Select(n => new NodeRedundancyState(
n.Node,
n.Role,
IsClusterLeader: n.Role == RedundancyRole.Primary,
IsDriverPrimary: n.Role == RedundancyRole.Primary,
AsOfUtc: DateTime.UtcNow))
.ToList(),
CorrelationId.NewId());
}