Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorWarmTests.cs
T
Joseph Doherty c569f80694 fix(717): Warm crash survivor stays Active through auto-down; verdict reads its own events
Two defects on the crash-failover path, found by the T13 live gate on an isolated docker pair:

1. When auto-down removed the unreachable Primary (~15 s after the survivor promoted), the
   snapshot's Primary left the unreachable set, so (3') failed and the survivor demoted to
   Standby (BadOutOfService, ServiceLevel 50) for ~15 s, until its own singleton published
   after the hand-over retries. A Primary that is Down, or that this node saw removed, now
   counts as gone (WarmRoleAuthority.SnapshotPrimaryGone). A member that is merely
   Leaving/Exiting, or that this node never saw, is not.

2. The verdict read Cluster.State / SelfMember (Akka's ClusterReadView, a separate subscriber
   to the same events), which can lag the event being handled. Live, the survivor stayed
   Standby 4.3 s after its own UnreachableMember event (promotion waited for the next health
   tick). The Warm verdict now uses a membership view built from this actor's own
   member/reachability subscription, falling back to the read view only before the first event.

Tests: WarmRoleAuthorityTests.A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active,
OpcUaPublishActorWarmTests.The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal
(synthetic events the read view never sees). Mutations: read-view-only verdict and the removed
arm switched off each fail the actor test. Runtime.Tests 1058 passed, 3 skipped.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 04:36:58 -05:00

283 lines
14 KiB
C#

using System.Collections.Concurrent;
using System.Text;
using Akka.Actor;
using Akka.TestKit;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
/// <summary>
/// Gitea #717 spec §4.1–§4.3 and §4.7 in <see cref="OpcUaPublishActor"/>: the Warm verdict from membership + the
/// accepted revisioned snapshot, the bands, and the transition step order — with a recording projection and
/// ServiceLevel publisher writing into ONE ordered log, so the invariant "an Active-band level is never visible over
/// projected data" is checked on the actual sequence.
/// </summary>
public sealed class OpcUaPublishActorWarmTests : RuntimeActorTestBase
{
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
private readonly ConcurrentQueue<string> _log = new();
private long _seq;
private NodeId Self
{
get
{
var cluster = Akka.Cluster.Cluster.Get(Sys);
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
}
}
private IActorRef Start(WarmRedundancySettings redundancy, LoggingPublisher? publisher = null) =>
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher ?? new LoggingPublisher(_log), localNode: Self, dbLess: true,
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: redundancy,
standbyProjection: new RecordingProjection(_log)));
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
new(new[]
{
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Self, DateTime.UtcNow),
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Other, DateTime.UtcNow),
},
CorrelationId.NewId(),
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
private string[] Steps => _log.ToArray();
private void AwaitLogEndsWith(params string[] tail) =>
AwaitAssert(() => Steps.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", Steps)), PresenceBudget,
cancellationToken: TestContext.Current.CancellationToken);
private static readonly string[] Promotion = ["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL:250"];
private static readonly string[] Demotion = ["SL:90", "delivery:True", "alarms:True", "proj:True", "queue:True"];
[Fact]
public void Promotion_and_demotion_run_in_the_spec_order()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
Steps.ShouldBe(Promotion, "boot is Standby; the first accepted snapshot promotes and nothing else ran");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith(Demotion);
AssertNoActiveLevelOverProjectedData();
}
/// <summary>Spec §4.7 invariant over the whole sequence: every published level ≥ 100 comes after the projection was
/// last opened, never while it is closed.</summary>
private void AssertNoActiveLevelOverProjectedData()
{
var projected = true; // a Warm node boots projected
foreach (var step in Steps)
{
if (step == "proj:True") projected = true;
else if (step == "proj:False") projected = false;
else if (step.StartsWith("SL:", StringComparison.Ordinal) && byte.Parse(step[3..]) >= 100)
projected.ShouldBeFalse($"an Active-band level was visible over projected data: {string.Join(" ", Steps)}");
}
}
[Fact]
public void An_unrevisioned_snapshot_never_makes_a_Warm_node_Active()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self, revisioned: false));
AwaitLogEndsWith("SL:90");
Steps.ShouldBe(new[] { "SL:90" });
}
/// <summary>Spec §4.2: after a newer snapshot names this node Primary, an OLDER incarnation's late publish naming the
/// peer is dropped — the node stays Active. Then a genuinely newer one demotes it (positive control).</summary>
[Fact]
public void An_older_revision_after_a_newer_one_is_dropped()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self, upNumber: 5));
AwaitLogEndsWith(Promotion);
actor.Tell(Snapshot(Other, upNumber: 4)); // a departed older singleton's late heartbeat
actor.Tell(Snapshot(Self, upNumber: 5)); // a fence: processed after the stale one
AwaitAssert(() => Steps.Count(s => s == "SL:250").ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
Steps.ShouldNotContain("delivery:True", "the stale snapshot must not demote");
actor.Tell(Snapshot(Other, upNumber: 6));
AwaitLogEndsWith(Demotion);
}
/// <summary>Spec §4.7: if the Active level cannot be published, the promotion stops before it and retries; the
/// Standby level stays visible meanwhile.</summary>
[Fact]
public void A_failed_Active_level_publish_halts_before_it_and_retries()
{
var publisher = new LoggingPublisher(_log) { FailNext = 1, FailLevel = 250 };
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
actor.Tell(Snapshot(Self));
AwaitAssert(() => Steps.ShouldContain("SL:250"), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
var log = Steps;
log.Take(6).ShouldBe(["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL-FAILED:250"]);
log.Last().ShouldBe("SL:250", "retried without a new snapshot");
}
/// <summary>A demotion continues when the Standby level publish fails: failing safe degrades the data.</summary>
[Fact]
public void A_failed_Standby_level_publish_still_degrades_the_data()
{
var publisher = new LoggingPublisher(_log);
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
publisher.FailNext = 1;
publisher.FailLevel = 90;
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL-FAILED:90", "delivery:True", "alarms:True", "proj:True", "queue:True");
}
[Fact]
public void A_burst_of_alternating_verdicts_converges_on_the_last()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
for (var i = 0; i < 20; i++) actor.Tell(Snapshot(i % 2 == 0 ? Self : Other));
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
AssertNoActiveLevelOverProjectedData();
}
/// <summary>
/// T13 live gate (2026-11-13), two findings on the crash path, driven here with synthetic cluster events that
/// Akka's read view (Cluster.State) never sees:
/// (1) the verdict reacts to the event it is handling, not to the read view, which can lag it (live: the survivor
/// stayed Standby 4.3 s after its own UnreachableMember event);
/// (2) when auto-down removes the unreachable Primary, the survivor stays Active (live: it demoted for ~15 s until
/// its own singleton published).
/// </summary>
[Fact]
public void The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
var other = MemberAt(Other);
actor.Tell(new Akka.Cluster.ClusterEvent.MemberUp(other));
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL:90");
actor.Tell(new Akka.Cluster.ClusterEvent.UnreachableMember(other));
AwaitLogEndsWith(Promotion);
actor.Tell(new Akka.Cluster.ClusterEvent.MemberRemoved(MemberAt(Other, Akka.Cluster.MemberStatus.Removed), Akka.Cluster.MemberStatus.Down));
actor.Tell(Snapshot(Other, upNumber: 1)); // the same accepted revision line, still naming the departed Primary
ExpectNoMsg(TimeSpan.FromMilliseconds(500), TestContext.Current.CancellationToken);
Steps.ShouldNotContain("delivery:True", "a Primary removed by auto-down is still gone; the survivor stays Active");
// Positive control: the departed Primary comes back Up and reachable, so the snapshot naming it demotes.
actor.Tell(new Akka.Cluster.ClusterEvent.MemberUp(other));
actor.Tell(new Akka.Cluster.ClusterEvent.ReachableMember(other));
actor.Tell(Snapshot(Other, upNumber: 1));
AwaitLogEndsWith(Demotion);
}
private static int _memberUid = 1000;
// Akka's Member.Create factories are internal; same reflection idiom as DriverHostActorHistorizationRoleViewTests.
private static readonly System.Reflection.MethodInfo MemberCreate = typeof(Akka.Cluster.Member).GetMethod(
"Create",
System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public,
binder: null,
types:
[
typeof(Akka.Cluster.UniqueAddress), typeof(int), typeof(Akka.Cluster.MemberStatus),
typeof(System.Collections.Immutable.ImmutableHashSet<string>), typeof(Akka.Util.AppVersion),
],
modifiers: null)
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found; Akka API changed");
private static Akka.Cluster.Member MemberAt(NodeId node, Akka.Cluster.MemberStatus status = Akka.Cluster.MemberStatus.Up)
{
var text = node.ToString();
var colon = text.LastIndexOf(':');
return (Akka.Cluster.Member)MemberCreate.Invoke(null,
[
new Akka.Cluster.UniqueAddress(
new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])), 7),
Interlocked.Increment(ref _memberUid),
status,
System.Collections.Immutable.ImmutableHashSet.Create("driver"),
Akka.Util.AppVersion.Zero,
])!;
}
/// <summary>The default-off pin at the actor: Unset never touches the projection and keeps today's levels.</summary>
[Fact]
public void Unset_never_drives_the_projection_and_keeps_todays_levels()
{
var actor = Start(WarmRedundancySettings.Unset);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL:240");
Steps.ShouldBe(new[] { "SL:250", "SL:240" });
}
[Fact]
public void A_deployed_cluster_mode_that_disagrees_with_the_local_mode_is_logged_and_counted()
{
var artifact = Encoding.UTF8.GetBytes(
"{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":\"None\"}],\"Nodes\":[]}");
DeploymentArtifact.ReadClusterRedundancyMode(artifact, "x:1").ShouldBe("None");
DeploymentArtifact.ReadClusterRedundancyMode(
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":1}]}"), "x:1").ShouldBe("Warm");
DeploymentArtifact.ReadClusterRedundancyMode(
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"a\",\"RedundancyMode\":\"None\"},{\"ClusterId\":\"b\",\"RedundancyMode\":\"Hot\"}],"
+ "\"Nodes\":[{\"NodeId\":\"x:1\",\"ClusterId\":\"b\"}]}"), "x:1").ShouldBe("Hot");
using var listener = new System.Diagnostics.Metrics.MeterListener();
long mismatches = 0;
listener.InstrumentPublished = (instrument, l) =>
{
if (ReferenceEquals(instrument, ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.RedundancyModeMismatch))
l.EnableMeasurementEvents(instrument);
};
listener.SetMeasurementEventCallback<long>((_, value, _, _) => Interlocked.Add(ref mismatches, value));
listener.Start();
var warmActor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
localNode: NodeId.Parse("x:1"), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm)));
EventFilter.Error(contains: "RedundancyMode for this node's cluster is None").ExpectOne(() =>
warmActor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), Artifact: artifact)),
TestContext.Current.CancellationToken);
Interlocked.Read(ref mismatches).ShouldBeGreaterThanOrEqualTo(1);
}
}
/// <summary>Writes each published level into the shared log; can be told to fail the next N publishes of a level.</summary>
internal sealed class LoggingPublisher(ConcurrentQueue<string> log) : IServiceLevelPublisher
{
public int FailNext { get; set; }
public byte FailLevel { get; set; }
public void Publish(byte serviceLevel)
{
if (FailNext > 0 && serviceLevel == FailLevel)
{
FailNext--;
log.Enqueue($"SL-FAILED:{serviceLevel}");
throw new InvalidOperationException("injected ServiceLevel publish failure");
}
log.Enqueue($"SL:{serviceLevel}");
}
}