c569f80694
Two defects on the crash-failover path, found by the T13 live gate on an isolated docker pair: 1. When auto-down removed the unreachable Primary (~15 s after the survivor promoted), the snapshot's Primary left the unreachable set, so (3') failed and the survivor demoted to Standby (BadOutOfService, ServiceLevel 50) for ~15 s, until its own singleton published after the hand-over retries. A Primary that is Down, or that this node saw removed, now counts as gone (WarmRoleAuthority.SnapshotPrimaryGone). A member that is merely Leaving/Exiting, or that this node never saw, is not. 2. The verdict read Cluster.State / SelfMember (Akka's ClusterReadView, a separate subscriber to the same events), which can lag the event being handled. Live, the survivor stayed Standby 4.3 s after its own UnreachableMember event (promotion waited for the next health tick). The Warm verdict now uses a membership view built from this actor's own member/reachability subscription, falling back to the read view only before the first event. Tests: WarmRoleAuthorityTests.A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active, OpcUaPublishActorWarmTests.The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal (synthetic events the read view never sees). Mutations: read-view-only verdict and the removed arm switched off each fail the actor test. Runtime.Tests 1058 passed, 3 skipped. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
283 lines
14 KiB
C#
283 lines
14 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Text;
|
|
using Akka.Actor;
|
|
using Akka.TestKit;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
|
|
|
|
/// <summary>
|
|
/// Gitea #717 spec §4.1–§4.3 and §4.7 in <see cref="OpcUaPublishActor"/>: the Warm verdict from membership + the
|
|
/// accepted revisioned snapshot, the bands, and the transition step order — with a recording projection and
|
|
/// ServiceLevel publisher writing into ONE ordered log, so the invariant "an Active-band level is never visible over
|
|
/// projected data" is checked on the actual sequence.
|
|
/// </summary>
|
|
public sealed class OpcUaPublishActorWarmTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
|
|
|
|
private readonly ConcurrentQueue<string> _log = new();
|
|
private long _seq;
|
|
|
|
private NodeId Self
|
|
{
|
|
get
|
|
{
|
|
var cluster = Akka.Cluster.Cluster.Get(Sys);
|
|
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
|
|
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
|
|
}
|
|
}
|
|
|
|
private IActorRef Start(WarmRedundancySettings redundancy, LoggingPublisher? publisher = null) =>
|
|
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
|
serviceLevel: publisher ?? new LoggingPublisher(_log), localNode: Self, dbLess: true,
|
|
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: redundancy,
|
|
standbyProjection: new RecordingProjection(_log)));
|
|
|
|
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
|
|
new(new[]
|
|
{
|
|
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
|
|
false, primary == Self, DateTime.UtcNow),
|
|
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
|
|
false, primary == Other, DateTime.UtcNow),
|
|
},
|
|
CorrelationId.NewId(),
|
|
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
|
|
|
|
private string[] Steps => _log.ToArray();
|
|
|
|
private void AwaitLogEndsWith(params string[] tail) =>
|
|
AwaitAssert(() => Steps.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", Steps)), PresenceBudget,
|
|
cancellationToken: TestContext.Current.CancellationToken);
|
|
|
|
private static readonly string[] Promotion = ["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL:250"];
|
|
private static readonly string[] Demotion = ["SL:90", "delivery:True", "alarms:True", "proj:True", "queue:True"];
|
|
|
|
[Fact]
|
|
public void Promotion_and_demotion_run_in_the_spec_order()
|
|
{
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitLogEndsWith(Promotion);
|
|
Steps.ShouldBe(Promotion, "boot is Standby; the first accepted snapshot promotes and nothing else ran");
|
|
|
|
actor.Tell(Snapshot(Other));
|
|
AwaitLogEndsWith(Demotion);
|
|
AssertNoActiveLevelOverProjectedData();
|
|
}
|
|
|
|
/// <summary>Spec §4.7 invariant over the whole sequence: every published level ≥ 100 comes after the projection was
|
|
/// last opened, never while it is closed.</summary>
|
|
private void AssertNoActiveLevelOverProjectedData()
|
|
{
|
|
var projected = true; // a Warm node boots projected
|
|
foreach (var step in Steps)
|
|
{
|
|
if (step == "proj:True") projected = true;
|
|
else if (step == "proj:False") projected = false;
|
|
else if (step.StartsWith("SL:", StringComparison.Ordinal) && byte.Parse(step[3..]) >= 100)
|
|
projected.ShouldBeFalse($"an Active-band level was visible over projected data: {string.Join(" ", Steps)}");
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void An_unrevisioned_snapshot_never_makes_a_Warm_node_Active()
|
|
{
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
|
actor.Tell(Snapshot(Self, revisioned: false));
|
|
AwaitLogEndsWith("SL:90");
|
|
Steps.ShouldBe(new[] { "SL:90" });
|
|
}
|
|
|
|
/// <summary>Spec §4.2: after a newer snapshot names this node Primary, an OLDER incarnation's late publish naming the
|
|
/// peer is dropped — the node stays Active. Then a genuinely newer one demotes it (positive control).</summary>
|
|
[Fact]
|
|
public void An_older_revision_after_a_newer_one_is_dropped()
|
|
{
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
|
actor.Tell(Snapshot(Self, upNumber: 5));
|
|
AwaitLogEndsWith(Promotion);
|
|
|
|
actor.Tell(Snapshot(Other, upNumber: 4)); // a departed older singleton's late heartbeat
|
|
actor.Tell(Snapshot(Self, upNumber: 5)); // a fence: processed after the stale one
|
|
AwaitAssert(() => Steps.Count(s => s == "SL:250").ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
|
|
Steps.ShouldNotContain("delivery:True", "the stale snapshot must not demote");
|
|
|
|
actor.Tell(Snapshot(Other, upNumber: 6));
|
|
AwaitLogEndsWith(Demotion);
|
|
}
|
|
|
|
/// <summary>Spec §4.7: if the Active level cannot be published, the promotion stops before it and retries; the
|
|
/// Standby level stays visible meanwhile.</summary>
|
|
[Fact]
|
|
public void A_failed_Active_level_publish_halts_before_it_and_retries()
|
|
{
|
|
var publisher = new LoggingPublisher(_log) { FailNext = 1, FailLevel = 250 };
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitAssert(() => Steps.ShouldContain("SL:250"), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
var log = Steps;
|
|
log.Take(6).ShouldBe(["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL-FAILED:250"]);
|
|
log.Last().ShouldBe("SL:250", "retried without a new snapshot");
|
|
}
|
|
|
|
/// <summary>A demotion continues when the Standby level publish fails: failing safe degrades the data.</summary>
|
|
[Fact]
|
|
public void A_failed_Standby_level_publish_still_degrades_the_data()
|
|
{
|
|
var publisher = new LoggingPublisher(_log);
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitLogEndsWith(Promotion);
|
|
publisher.FailNext = 1;
|
|
publisher.FailLevel = 90;
|
|
actor.Tell(Snapshot(Other));
|
|
AwaitLogEndsWith("SL-FAILED:90", "delivery:True", "alarms:True", "proj:True", "queue:True");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_burst_of_alternating_verdicts_converges_on_the_last()
|
|
{
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
|
for (var i = 0; i < 20; i++) actor.Tell(Snapshot(i % 2 == 0 ? Self : Other));
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitLogEndsWith(Promotion);
|
|
AssertNoActiveLevelOverProjectedData();
|
|
}
|
|
|
|
/// <summary>
|
|
/// T13 live gate (2026-11-13), two findings on the crash path, driven here with synthetic cluster events that
|
|
/// Akka's read view (Cluster.State) never sees:
|
|
/// (1) the verdict reacts to the event it is handling, not to the read view, which can lag it (live: the survivor
|
|
/// stayed Standby 4.3 s after its own UnreachableMember event);
|
|
/// (2) when auto-down removes the unreachable Primary, the survivor stays Active (live: it demoted for ~15 s until
|
|
/// its own singleton published).
|
|
/// </summary>
|
|
[Fact]
|
|
public void The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal()
|
|
{
|
|
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
|
var other = MemberAt(Other);
|
|
actor.Tell(new Akka.Cluster.ClusterEvent.MemberUp(other));
|
|
actor.Tell(Snapshot(Other));
|
|
AwaitLogEndsWith("SL:90");
|
|
|
|
actor.Tell(new Akka.Cluster.ClusterEvent.UnreachableMember(other));
|
|
AwaitLogEndsWith(Promotion);
|
|
|
|
actor.Tell(new Akka.Cluster.ClusterEvent.MemberRemoved(MemberAt(Other, Akka.Cluster.MemberStatus.Removed), Akka.Cluster.MemberStatus.Down));
|
|
actor.Tell(Snapshot(Other, upNumber: 1)); // the same accepted revision line, still naming the departed Primary
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(500), TestContext.Current.CancellationToken);
|
|
Steps.ShouldNotContain("delivery:True", "a Primary removed by auto-down is still gone; the survivor stays Active");
|
|
|
|
// Positive control: the departed Primary comes back Up and reachable, so the snapshot naming it demotes.
|
|
actor.Tell(new Akka.Cluster.ClusterEvent.MemberUp(other));
|
|
actor.Tell(new Akka.Cluster.ClusterEvent.ReachableMember(other));
|
|
actor.Tell(Snapshot(Other, upNumber: 1));
|
|
AwaitLogEndsWith(Demotion);
|
|
}
|
|
|
|
private static int _memberUid = 1000;
|
|
|
|
// Akka's Member.Create factories are internal; same reflection idiom as DriverHostActorHistorizationRoleViewTests.
|
|
private static readonly System.Reflection.MethodInfo MemberCreate = typeof(Akka.Cluster.Member).GetMethod(
|
|
"Create",
|
|
System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public,
|
|
binder: null,
|
|
types:
|
|
[
|
|
typeof(Akka.Cluster.UniqueAddress), typeof(int), typeof(Akka.Cluster.MemberStatus),
|
|
typeof(System.Collections.Immutable.ImmutableHashSet<string>), typeof(Akka.Util.AppVersion),
|
|
],
|
|
modifiers: null)
|
|
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found; Akka API changed");
|
|
|
|
private static Akka.Cluster.Member MemberAt(NodeId node, Akka.Cluster.MemberStatus status = Akka.Cluster.MemberStatus.Up)
|
|
{
|
|
var text = node.ToString();
|
|
var colon = text.LastIndexOf(':');
|
|
return (Akka.Cluster.Member)MemberCreate.Invoke(null,
|
|
[
|
|
new Akka.Cluster.UniqueAddress(
|
|
new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])), 7),
|
|
Interlocked.Increment(ref _memberUid),
|
|
status,
|
|
System.Collections.Immutable.ImmutableHashSet.Create("driver"),
|
|
Akka.Util.AppVersion.Zero,
|
|
])!;
|
|
}
|
|
|
|
/// <summary>The default-off pin at the actor: Unset never touches the projection and keeps today's levels.</summary>
|
|
[Fact]
|
|
public void Unset_never_drives_the_projection_and_keeps_todays_levels()
|
|
{
|
|
var actor = Start(WarmRedundancySettings.Unset);
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitLogEndsWith("SL:250");
|
|
actor.Tell(Snapshot(Other));
|
|
AwaitLogEndsWith("SL:240");
|
|
Steps.ShouldBe(new[] { "SL:250", "SL:240" });
|
|
}
|
|
|
|
[Fact]
|
|
public void A_deployed_cluster_mode_that_disagrees_with_the_local_mode_is_logged_and_counted()
|
|
{
|
|
var artifact = Encoding.UTF8.GetBytes(
|
|
"{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":\"None\"}],\"Nodes\":[]}");
|
|
DeploymentArtifact.ReadClusterRedundancyMode(artifact, "x:1").ShouldBe("None");
|
|
DeploymentArtifact.ReadClusterRedundancyMode(
|
|
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":1}]}"), "x:1").ShouldBe("Warm");
|
|
DeploymentArtifact.ReadClusterRedundancyMode(
|
|
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"a\",\"RedundancyMode\":\"None\"},{\"ClusterId\":\"b\",\"RedundancyMode\":\"Hot\"}],"
|
|
+ "\"Nodes\":[{\"NodeId\":\"x:1\",\"ClusterId\":\"b\"}]}"), "x:1").ShouldBe("Hot");
|
|
|
|
using var listener = new System.Diagnostics.Metrics.MeterListener();
|
|
long mismatches = 0;
|
|
listener.InstrumentPublished = (instrument, l) =>
|
|
{
|
|
if (ReferenceEquals(instrument, ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.RedundancyModeMismatch))
|
|
l.EnableMeasurementEvents(instrument);
|
|
};
|
|
listener.SetMeasurementEventCallback<long>((_, value, _, _) => Interlocked.Add(ref mismatches, value));
|
|
listener.Start();
|
|
|
|
var warmActor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
|
localNode: NodeId.Parse("x:1"), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm)));
|
|
EventFilter.Error(contains: "RedundancyMode for this node's cluster is None").ExpectOne(() =>
|
|
warmActor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), Artifact: artifact)),
|
|
TestContext.Current.CancellationToken);
|
|
Interlocked.Read(ref mismatches).ShouldBeGreaterThanOrEqualTo(1);
|
|
}
|
|
}
|
|
|
|
/// <summary>Writes each published level into the shared log; can be told to fail the next N publishes of a level.</summary>
|
|
internal sealed class LoggingPublisher(ConcurrentQueue<string> log) : IServiceLevelPublisher
|
|
{
|
|
public int FailNext { get; set; }
|
|
public byte FailLevel { get; set; }
|
|
|
|
public void Publish(byte serviceLevel)
|
|
{
|
|
if (FailNext > 0 && serviceLevel == FailLevel)
|
|
{
|
|
FailNext--;
|
|
log.Enqueue($"SL-FAILED:{serviceLevel}");
|
|
throw new InvalidOperationException("injected ServiceLevel publish failure");
|
|
}
|
|
|
|
log.Enqueue($"SL:{serviceLevel}");
|
|
}
|
|
}
|