feat(717): Warm non-transparent redundancy, default-off (T1-T10)

OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start).
- T1 options + validator + effective-URI check + boot banner; Warm needs the driver role.
- T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only
  ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None
  and the SDK's orphan i=11314 deleted; Unset untouched.
- T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older
  revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active.
- T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable.
- T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec
  4.7 order (Active level last and retried; Standby level first, data degraded regardless).
- T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState.
- T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by
  (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's
  cached copy is never mutated.
- T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters.
- T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion
  refresh via ISubscription.ConditionRefresh2Async on every live event item.
- T10 deploy-time RedundancyMode cross-check + six instruments.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:26:36 -05:00
parent dcff5ae994
commit 7829b937d1
27 changed files with 3056 additions and 29 deletions
@@ -0,0 +1,154 @@
using Akka.Actor;
using Akka.Cluster;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// Gitea #717 spec §4.2 (ordered snapshots) and §4.3 (Warm bands): the revision the singleton stamps, its order,
/// the subscriber gate, the wire shape of the extended message, and the Warm ServiceLevel truth table.
/// </summary>
public sealed class WarmRedundancyRevisionTests : ControlPlaneActorTestBase
{
[Fact]
public void The_singleton_stamps_an_increasing_revision_on_every_publish_heartbeats_included()
{
var probe = CreateTestProbe();
Sys.ActorOf(RedundancyStateActor.Props(msg => probe.Ref.Tell(msg), TimeSpan.FromMilliseconds(200)));
var first = probe.ExpectMsg<RedundancyStateChanged>(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);
var second = probe.ExpectMsg<RedundancyStateChanged>(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);
first.Revision.ShouldNotBeNull();
second.Revision.ShouldNotBeNull();
second.Revision!.CompareTo(first.Revision).ShouldBeGreaterThan(0);
second.Revision.Sequence.ShouldBe(first.Revision!.Sequence + 1);
first.Revision.PublisherUpNumber.ShouldBeGreaterThan(0L, "the publisher's Akka up-number");
}
/// <summary>Akka.NET keeps <c>Member.UpNumber</c> internal; an upgrade that renames it must fail here, not degrade
/// every revision to up-number 0 silently.</summary>
[Fact]
public void Akka_member_up_number_is_readable()
{
AwaitCondition(() => Akka.Cluster.Cluster.Get(Sys).SelfMember.Status == MemberStatus.Up, TimeSpan.FromSeconds(10),
cancellationToken: TestContext.Current.CancellationToken);
RedundancyStateActor.ReadUpNumber(Akka.Cluster.Cluster.Get(Sys).SelfMember).ShouldNotBeNull().ShouldBeGreaterThan(0);
}
[Fact]
public void A_singleton_restarted_in_the_same_process_outranks_its_predecessor()
{
var a = RedundancyStateActor.NextIncarnationTick();
var b = RedundancyStateActor.NextIncarnationTick();
b.ShouldBeGreaterThan(a, "strictly increasing even within one tick");
new SnapshotRevision(1, b, 1).CompareTo(new SnapshotRevision(1, a, 500)).ShouldBeGreaterThan(0);
}
[Fact]
public void Revisions_order_by_up_number_then_incarnation_then_sequence()
{
var baseline = new SnapshotRevision(2, 100, 50);
new SnapshotRevision(3, 1, 1).CompareTo(baseline).ShouldBeGreaterThan(0, "a handover to a younger member outranks");
new SnapshotRevision(1, 999, 999).CompareTo(baseline).ShouldBeLessThan(0, "a departed older member's late publish loses");
new SnapshotRevision(2, 101, 1).CompareTo(baseline).ShouldBeGreaterThan(0);
new SnapshotRevision(2, 100, 51).CompareTo(baseline).ShouldBeGreaterThan(0);
new SnapshotRevision(2, 100, 50).CompareTo(baseline).ShouldBe(0);
}
[Fact]
public void The_gate_accepts_only_newer_revisions_and_never_advances_on_an_unrevisioned_snapshot()
{
var gate = new SnapshotRevisionGate();
gate.Offer(Snap(null)).ShouldBe(SnapshotRejection.Unrevisioned);
gate.LastAccepted.ShouldBeNull();
gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.None);
gate.Offer(Snap(new(1, 9, 9))).ShouldBe(SnapshotRejection.OlderRevision, "an old incarnation's late heartbeat");
gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.OlderRevision, "a duplicate");
gate.Offer(Snap(new(2, 1, 6))).ShouldBe(SnapshotRejection.None);
gate.LastAccepted.ShouldBe(new SnapshotRevision(2, 1, 6));
}
[Fact]
public void The_extended_message_round_trips_and_a_pre_717_payload_decodes_with_no_revision()
{
var msg = Snap(new SnapshotRevision(3, 42, 7));
var serializer = Sys.Serialization.FindSerializerFor(msg);
var manifest = Akka.Serialization.Serialization.ManifestFor(serializer, msg);
var round = (RedundancyStateChanged)Sys.Serialization.Deserialize(serializer.ToBinary(msg), serializer.Identifier, manifest);
round.Revision.ShouldBe(msg.Revision);
round.Nodes.Single().NodeId.ShouldBe(msg.Nodes.Single().NodeId);
// A pre-#717 publisher's payload: the same message without the field.
var legacy = Snap(null);
var legacyBytes = serializer.ToBinary(legacy);
var text = System.Text.Encoding.UTF8.GetString(legacyBytes);
var stripped = System.Text.Encoding.UTF8.GetBytes(text.Replace(",\"Revision\":null", string.Empty, StringComparison.Ordinal));
var decoded = (RedundancyStateChanged)Sys.Serialization.Deserialize(stripped, serializer.Identifier, manifest);
decoded.Revision.ShouldBeNull();
decoded.Nodes.Count.ShouldBe(1);
}
private static RedundancyStateChanged Snap(SnapshotRevision? revision) =>
new(new[] { new NodeRedundancyState(NodeId.Parse("a:1"), RedundancyRole.Primary, true, true, DateTime.UtcNow) },
CorrelationId.NewId(), revision);
}
/// <summary>Gitea #717 spec §4.3: the Warm ServiceLevel bands (pure; no actor system).</summary>
public sealed class WarmServiceLevelBandTests
{
public static IEnumerable<object[]> AllInputs()
{
foreach (var member in new[] { MemberStatus.Up, MemberStatus.Joining, MemberStatus.Leaving, MemberStatus.Down })
foreach (var db in new[] { true, false })
foreach (var probe in new[] { true, false })
foreach (var stale in new[] { true, false })
foreach (var listening in new[] { true, false })
yield return [member, db, probe, stale, listening];
}
[Theory]
[MemberData(nameof(AllInputs))]
public void Every_standby_level_is_below_every_active_level_that_serves_data(
MemberStatus member, bool db, bool probe, bool stale, bool listening)
{
var h = new NodeHealthInputs(member, db, probe, stale, IsDriverPrimary: false, listening);
var standby = ServiceLevelCalculator.ComputeWarm(h, isActive: false);
var active = ServiceLevelCalculator.ComputeWarm(h, isActive: true);
standby.ShouldBeLessThanOrEqualTo(ServiceLevelCalculator.WarmStandbyHealthy);
if (active > 0) active.ShouldBeGreaterThanOrEqualTo(ServiceLevelCalculator.WarmActiveCritical);
if (active > 0) standby.ShouldBeLessThan(active);
}
[Theory]
[InlineData(true, true, false, true, 250)]
[InlineData(true, true, true, true, 200)]
[InlineData(false, true, true, true, 100)]
[InlineData(false, true, false, true, 100)]
[InlineData(true, false, false, true, 100)]
[InlineData(true, true, false, false, 90)]
[InlineData(true, true, true, false, 50)]
[InlineData(false, true, false, false, 50)]
[InlineData(true, false, false, false, 50)]
public void Warm_bands_match_the_spec_table(bool db, bool probe, bool stale, bool active, int expected)
{
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, db, probe, stale, IsDriverPrimary: false), active)
.ShouldBe((byte)expected);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Down_or_a_failed_endpoint_is_zero_in_either_role(bool active)
{
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Down, true, true, false, false), active).ShouldBe((byte)0);
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, true, true, false, false, EndpointListening: false), active)
.ShouldBe((byte)0);
}
}
@@ -132,3 +132,38 @@ public sealed class OpcUaApplicationHostOptionsValidatorTests
if (!valid) result.Failures!.ShouldContain(f => f.Contains("OpcUa:Discovery"));
}
}
/// <summary>Gitea #717 spec §3: the redundancy rules the startup validator and the driver-role guard enforce.</summary>
public sealed class OpcUaRedundancyValidationTests
{
private static readonly OpcUaApplicationHostOptionsValidator Sut = new();
[Theory]
[InlineData("Unset", new string[0], true)]
[InlineData("None", new string[0], true)]
[InlineData("None", new[] { "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:peer" }, true)]
[InlineData("Warm", new string[0], false)]
[InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:OtOpcUa" }, false)]
[InlineData("Hot", new[] { "urn:peer" }, false)]
[InlineData("Bogus", new string[0], false)]
public void Redundancy_matrix(string mode, string[] peers, bool valid)
{
var result = Sut.Validate(null, new OpcUaApplicationHostOptions
{
Redundancy = new OpcUaRedundancyOptions { Mode = mode },
PeerApplicationUris = peers.ToList(),
});
result.Succeeded.ShouldBe(valid, result.FailureMessage);
}
[Fact]
public void Warm_on_a_node_without_the_driver_role_fails_start()
{
Should.Throw<InvalidOperationException>(() => OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: false));
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: true);
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(null, hasDriver: false);
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("None", hasDriver: false);
}
}
@@ -44,6 +44,9 @@ internal sealed class ServerEventRig : IAsyncDisposable
public string EndpointUrl => $"opc.tcp://127.0.0.1:{_port}/OtOpcUa";
/// <summary>The configured <c>OpcUa:ApplicationUri</c> (the effective one: the rig loads no XML).</summary>
public string ApplicationUri => _hostOptions.ApplicationUri;
public OtOpcUaNodeManager Nm => _server.NodeManager!;
internal OtOpcUaSdkServer Server => _server;
@@ -57,7 +60,12 @@ internal sealed class ServerEventRig : IAsyncDisposable
public SdkAddressSpaceSink Sink { get; private set; } = null!;
public static async Task<ServerEventRig> StartAsync(string name, bool encrypted, CancellationToken ct)
public static Task<ServerEventRig> StartAsync(string name, bool encrypted, CancellationToken ct) =>
StartAsync(name, encrypted, ct, configure: null);
/// <summary>Starts the rig, letting the caller adjust the host options first (Gitea #717: redundancy mode, peers).</summary>
public static async Task<ServerEventRig> StartAsync(
string name, bool encrypted, CancellationToken ct, Action<OpcUaApplicationHostOptions>? configure)
{
var rig = new ServerEventRig(Path.Combine(Path.GetTempPath(), $"otopcua-{name}-{Guid.NewGuid():N}"));
rig._hostOptions = new OpcUaApplicationHostOptions
@@ -71,6 +79,7 @@ internal sealed class ServerEventRig : IAsyncDisposable
: [OpcUaSecurityProfile.None],
AutoAcceptUntrustedClientCertificates = true,
};
configure?.Invoke(rig._hostOptions);
(rig._host, rig._server) = await OpcUaHostStart.StartAsync(
port =>
{
@@ -0,0 +1,616 @@
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Gitea #717 over the wire, against the real <see cref="OtOpcUaSdkServer"/> and node manager: the redundancy
/// metadata per mode (T2), the Standby read projection and write refusal (T6), the D4 first-delivery decision record
/// across the three demotion timings T0 established (T7), the live-item queue on promotion/demotion (T8), and the
/// alarm suppression, refusal and promotion refresh (T9). Transitions are driven through the server's own
/// <see cref="IStandbyProjection"/> in the spec §4.7 step order; the actor that owns that order has its own tests.
/// </summary>
[Trait("Category", "Integration")]
public sealed class WarmRedundancyWireTests
{
private const string Peer = "urn:OtOpcUa:peer-717";
private const NodePermissions All =
NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead
| NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve
| NodePermissions.MethodCall | NodePermissions.WriteOperate;
private static Task<ServerEventRig> StartModeAsync(string name, string mode, CancellationToken ct) =>
ServerEventRig.StartAsync(name, encrypted: false, ct, o =>
{
o.Redundancy = new OpcUaRedundancyOptions { Mode = mode };
if (mode == "Warm") o.PeerApplicationUris = [Peer];
});
/// <summary>Spec §4.7 promotion, in order (the actor's sequence, minus the ServiceLevel publish).</summary>
internal static void Promote(IStandbyProjection p)
{
p.SetDataProjection(false);
p.QueueCurrentToLiveItems(false);
p.SetStandbyDelivery(false);
p.SetAlarmSuppression(false);
p.RefreshAllEventItems();
}
/// <summary>Spec §4.7 demotion, in order (after the Standby ServiceLevel publish).</summary>
internal static void Demote(IStandbyProjection p)
{
p.SetStandbyDelivery(true);
p.SetAlarmSuppression(true);
p.SetDataProjection(true);
p.QueueCurrentToLiveItems(true);
}
private static CancellationToken Deadline(int seconds = 90)
{
var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
cts.CancelAfter(TimeSpan.FromSeconds(seconds));
return cts.Token;
}
private static RequestHeader H() => new() { Timestamp = DateTimeUtc.Now, TimeoutHint = 10_000 };
private static NodeId Raw(ISession session, string id) =>
new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
private static async Task<DataValue> ReadAsync(ISession session, NodeId node, CancellationToken ct, uint attribute = Attributes.Value,
string? indexRange = null)
{
var r = await ((SessionClient)session).ReadAsync(H(), 0, TimestampsToReturn.Both,
new[] { new ReadValueId { NodeId = node, AttributeId = attribute, IndexRange = indexRange } }, ct);
return r.Results[0];
}
// ------------------------------------------------------------------ T2 metadata
[Fact]
public async Task Warm_advertises_non_transparent_metadata_self_first_and_boots_at_ServiceLevel_1()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Meta", "Warm", ct);
using var session = await rig.ConnectAsync(anonymous: true, ct);
Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
.ShouldBe((int)RedundancySupport.Warm);
var uris = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
uris.StatusCode.ShouldBe(StatusCodes.Good);
((string[])uris.ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.WarmBootServiceLevel);
var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
{
new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
}, ct);
browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.NonTransparentRedundancyType));
var write = await ((SessionClient)session).WriteAsync(H(), new[]
{
new WriteValue { NodeId = VariableIds.Server_ServerRedundancy_ServerUriArray, AttributeId = Attributes.Value,
Value = new DataValue(new Variant(new[] { "x" })) },
}, ct);
StatusCode.IsBad(write.Results[0]).ShouldBeTrue($"ServerUriArray is read-only, got {write.Results[0]}");
}
[Fact]
public async Task None_advertises_RedundancySupport_None_and_ServerUriArray_is_absent_even_by_NodeId()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717None", "None", ct);
using var session = await rig.ConnectAsync(anonymous: true, ct);
Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
.ShouldBe((int)RedundancySupport.None);
(await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct)).StatusCode
.ShouldBe(StatusCodes.BadNodeIdUnknown, "T0a: the SDK's orphan i=11314 is deleted under None");
var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
{
new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
}, ct);
browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.ServerRedundancyType));
((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri });
}
/// <summary>The default-off pin: an Unset server is today's server — SDK-default RedundancySupport, the SDK's own
/// (orphan) i=11314, ServerRedundancyType, the #885 boot level 0, Good data with no projection, and no Warm hooks.</summary>
[Fact]
public async Task Unset_is_todays_server_exactly()
{
var ct = Deadline();
await using var rig = await ServerEventRig.StartAsync("Warm717Unset", encrypted: false, ct);
rig.Server.RedundancyMode.ShouldBe(OpcUaRedundancyMode.Unset);
rig.Server.CreateStandbyProjection().ShouldBeNull();
rig.Nm.Standby.WarmMode.ShouldBeFalse();
var id = rig.EnsureRawVariable("u", "Double", writable: false);
rig.Sink.WriteValue(id, 4.5, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var support = await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct);
support.StatusCode.ShouldBe(StatusCodes.Good);
Convert.ToInt32(support.ClrValue()).ShouldBe((int)RedundancySupport.None, "the SDK default, never written by us");
var orphan = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
orphan.StatusCode.ShouldBe(StatusCodes.Good, "Unset leaves the SDK's orphan i=11314 exactly as the SDK builds it");
Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel);
var value = await ReadAsync(session, Raw(session, id), ct);
value.StatusCode.ShouldBe(StatusCodes.Good);
value.ClrValue().ShouldBe(4.5);
}
// ------------------------------------------------------------------ T6 read projection + write refusal
[Fact]
public async Task Standby_projects_ordinary_tag_reads_but_not_metadata_and_promotion_reads_the_latest_stored_value()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Read", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection().ShouldNotBeNull();
var raw = rig.EnsureRawVariable("t1", "Double", writable: false);
rig.Sink.EnsureVariable("EqA/t1ref", null, "t1ref", "Double", writable: false, AddressSpaceRealm.Uns);
var arr = $"{RawDeviceFolder}/arr";
rig.Sink.EnsureVariable(arr, RawDeviceFolder, "arr", "Int32", writable: false, AddressSpaceRealm.Raw, isArray: true);
rig.EnsureEquipment();
rig.Sink.EnsureVariable("EqA/vt", "EqA", "vt", "Double", writable: false, AddressSpaceRealm.Uns);
rig.MaterialiseScripted();
var ts = DateTime.UtcNow;
rig.Sink.WriteValue(raw, 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
rig.Sink.WriteValue("EqA/t1ref", 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
rig.Sink.WriteValue("EqA/vt", 2.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
rig.Sink.WriteValue(arr, new[] { 1, 2, 3 }, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
using var session = await rig.ConnectAsync(anonymous: true, ct);
foreach (var node in new[] { Raw(session, raw), UnsNodeId(session, "EqA/t1ref"), UnsNodeId(session, "EqA/vt") })
{
var dv = await ReadAsync(session, node, ct);
dv.StatusCode.ShouldBe(StatusCodes.BadOutOfService, node.ToString());
dv.WrappedValue.IsNull.ShouldBeTrue();
(dv.ServerTimestamp != DateTimeUtc.MinValue).ShouldBeTrue();
}
(await ReadAsync(session, Raw(session, arr), ct, indexRange: "1")).StatusCode.ShouldBe(StatusCodes.BadOutOfService);
(await ReadAsync(session, Raw(session, raw), ct, Attributes.DisplayName)).StatusCode.ShouldBe(StatusCodes.Good);
(await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).StatusCode.ShouldBe(StatusCodes.Good);
(await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).StatusCode.ShouldBe(StatusCodes.Good);
var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
(await ReadAsync(session, severity, ct)).StatusCode.ShouldBe(StatusCodes.Good, "a condition child is not an ordinary tag");
// Acquisition continues while projected; promotion reads the latest stored value with no new driver publish.
rig.Sink.WriteValue(raw, 9.25, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
var after = await ReadAsync(session, Raw(session, raw), ct);
after.StatusCode.ShouldBe(StatusCodes.Good);
after.ClrValue().ShouldBe(9.25);
var ranged = await ReadAsync(session, Raw(session, arr), ct, indexRange: "1");
ranged.StatusCode.ShouldBe(StatusCodes.Good);
((int[])ranged.ClrValue()!).ShouldBe(new[] { 2 }, "an IndexRange read reaches the stored array once promoted");
}
[Fact]
public async Task Standby_projection_never_leaks_an_ACL_denied_node()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Acl", "Warm", ct);
var raw = rig.EnsureRawVariable("secret", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Enforce(true, [Key()]); // anonymous holds no grant
using var session = await rig.ConnectAsync(anonymous: true, ct);
(await ReadAsync(session, Raw(session, raw), ct)).StatusCode.ShouldBe(StatusCodes.BadUserAccessDenied);
}
private sealed class RecordingWriteGateway : IOpcUaNodeWriteGateway
{
public int Count;
public Task<NodeWriteOutcome> WriteAsync(string nodeId, object? value, AddressSpaceRealm realm, CancellationToken ct)
{
Interlocked.Increment(ref Count);
return Task.FromResult(new NodeWriteOutcome(true, null));
}
}
[Theory]
[InlineData("Warm", true)]
[InlineData("Warm", false)]
[InlineData("Unset", true)]
public async Task A_write_on_a_Warm_standby_is_refused_BadInvalidState_and_never_reaches_the_gateway(string mode, bool standby)
{
var ct = Deadline();
await using var rig = await StartModeAsync($"Warm717Write{mode}{standby}", mode, ct);
var gateway = new RecordingWriteGateway();
rig.Nm.NodeWriteGateway = gateway;
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
var raw = rig.EnsureRawVariable("w", "Double", writable: true);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
if (mode == "Warm" && !standby) Promote(rig.Server.CreateStandbyProjection()!);
// Unset with "standby": a node that is not the Primary takes today's optimistic path (no gate wired here).
using var session = await rig.ConnectAsync(anonymous: false, ct);
var result = await ((SessionClient)session).WriteAsync(H(), new[]
{
new WriteValue { NodeId = Raw(session, raw), AttributeId = Attributes.Value, Value = new DataValue(new Variant(2.0)) },
}, ct);
if (mode == "Warm" && standby)
{
result.Results[0].ShouldBe(StatusCodes.BadInvalidState);
await Task.Delay(200, ct);
gateway.Count.ShouldBe(0, "a refused write never reaches the driver");
}
else
{
result.Results[0].ShouldBe(StatusCodes.Good);
await RecordedEvents.WaitUntilAsync(() => Volatile.Read(ref gateway.Count) == 1);
}
}
// ------------------------------------------------------------------ T9 Part 9 method refusal
[Theory]
[InlineData("Warm", true, "Acknowledge")]
[InlineData("Warm", true, "OneShotShelve")]
[InlineData("Warm", false, "Acknowledge")]
[InlineData("Unset", true, "Acknowledge")]
public async Task A_Part9_method_on_a_Warm_standby_is_refused_BadInvalidState_before_any_state_change(
string mode, bool standby, string method)
{
var ct = Deadline();
await using var rig = await StartModeAsync($"Warm717Method{mode}{standby}{method}", mode, ct);
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseScripted();
var routed = 0;
rig.Nm.AlarmCommandRouter = _ => Interlocked.Increment(ref routed);
var projection = rig.Server.CreateStandbyProjection();
if (projection is not null) Promote(projection); // project the condition while Active so it has an EventId
rig.Project(native: false, ConditionMethodEventTests.PreState(method, native: false) with { Message = "base" });
if (projection is not null && standby) Demote(projection);
using var session = await rig.ConnectAsync(anonymous: false, ct);
var eventId = await ReadEventIdAsync(session, ct);
var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), method, eventId, ct);
if (mode == "Warm" && standby)
{
(await Should.ThrowAsync<ServiceResultException>(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState);
routed.ShouldBe(0, "nothing is routed to the engine");
var acked = await TranslateAsync(session, ConditionNodeId(session, native: false), "AckedState/Id", ct);
(await ReadAsync(session, acked, ct)).ClrValue().ShouldBe(false, "no condition state changed");
}
else
{
await call;
routed.ShouldBe(1);
}
}
private static async Task<byte[]> ReadEventIdAsync(ISession session, CancellationToken ct)
{
var eventId = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.EventId, ct);
return ((ByteString)(await ReadAsync(session, eventId, ct)).WrappedValue.AsBoxedObject()!).ToArray()!;
}
private static async Task<NodeId> TranslateAsync(ISession session, NodeId start, string path, CancellationToken ct)
{
var elements = path.Split('/').Select(name => new RelativePathElement
{
ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, TargetName = new QualifiedName(name),
}).ToArray();
var r = await ((SessionClient)session).TranslateBrowsePathsToNodeIdsAsync(H(), new[]
{
new BrowsePath { StartingNode = start, RelativePath = new RelativePath { Elements = elements } },
}, ct);
r.Results[0].StatusCode.ShouldBe(StatusCodes.Good, path);
return ExpandedNodeId.ToNodeId(r.Results[0].Targets[0].TargetId, session.NamespaceUris);
}
// ------------------------------------------------------------------ T9 alarm suppression + refresh
[Fact]
public async Task Standby_suppresses_new_alarm_events_on_every_item_kind_and_promotion_refreshes_them_all()
{
var ct = Deadline(120);
await using var rig = await StartModeAsync("Warm717Alarms", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseNative();
rig.MaterialiseScripted();
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
var folderItem = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA));
var deviceItem = RecordedEvents.AddEventItem(subscription,
new NodeId(RawDeviceFolder, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri)));
await subscription.ApplyChangesAsync(ct);
// While Standby (boot state): a new transition reaches no item, but the condition node is current.
rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "while-standby"));
// A condition raised and cleared (Retain=false) while Standby is not replayed by the promotion refresh.
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash"));
rig.Project(native: false, new AlarmConditionSnapshot(Active: false, Acknowledged: true, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash-cleared"));
await Task.Delay(1000, ct);
foreach (var item in new[] { serverItem, folderItem, deviceItem })
{
events.Has(item, "while-standby").ShouldBeFalse($"no alarm event on {item.StartNodeId} while Standby");
events.Has(item, "flash").ShouldBeFalse();
}
// D5: a client ConditionRefresh on the Standby returns Start, the retained condition, End.
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
events.Has(serverItem, "while-standby").ShouldBeTrue("the Standby's refresh carries the retained condition");
events.Has(serverItem, "flash-cleared").ShouldBeFalse("a non-retained condition is not replayed");
foreach (var item in new[] { folderItem, deviceItem })
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
var countsBefore = new[] { serverItem, folderItem, deviceItem }.ToDictionary(i => i, i => events.Of(i).Count);
Promote(projection);
foreach (var item in new[] { serverItem, folderItem, deviceItem })
{
await RecordedEvents.WaitUntilAsync(() => events.Of(item).Skip(countsBefore[item])
.Any(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType));
var stream = events.Of(item).Skip(countsBefore[item]).ToList();
var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
start.ShouldBeGreaterThanOrEqualTo(0, $"RefreshStart on {item.StartNodeId}");
var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
replayed.ShouldContain("while-standby", $"promotion refresh replays the retained condition on {item.StartNodeId}");
replayed.ShouldNotContain("flash-cleared");
}
// After promotion new events flow, and the Standby-stamped EventId acknowledges.
rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 800, Message: "after"));
await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "after") && events.Has(folderItem, "after") && events.Has(deviceItem, "after"));
await subscription.DeleteAsync(true, ct);
}
[Fact]
public async Task A_Standby_stamped_EventId_is_accepted_by_Acknowledge_after_promotion()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717AckAfter", "Warm", ct);
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseScripted();
var routed = new List<AlarmCommand>();
rig.Nm.AlarmCommandRouter = cmd => { lock (routed) routed.Add(cmd); };
rig.Project(native: false, ConditionMethodEventTests.PreState("Acknowledge", native: false) with { Message = "stamped" });
using var session = await rig.ConnectAsync(anonymous: false, ct);
var stamped = await ReadEventIdAsync(session, ct); // stamped while Standby, never delivered
Promote(rig.Server.CreateStandbyProjection()!);
await ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), "Acknowledge", stamped, ct);
lock (routed) routed.ShouldHaveSingleItem().Operation.ShouldBe("Acknowledge");
}
// ------------------------------------------------------------------ T8 live items on transition
[Fact]
public async Task Demotion_queues_one_BadOutOfService_then_silence_and_promotion_queues_the_current_value_through_a_deadband()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Live", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("live", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
await AddItemAsync(client, sub, Raw(session, raw), 2, 10, ct, new DataChangeFilter
{
Trigger = DataChangeTrigger.StatusValue, DeadbandType = (uint)DeadbandType.Absolute, DeadbandValue = 1000,
});
(await CollectAsync(client, ct, TimeSpan.FromSeconds(1))).Where(x => x.Handle is 1 or 2).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
Demote(projection);
rig.Sink.WriteValue(raw, 2.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 3.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
var demoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
demoted.Count(x => x.Handle == 1).ShouldBe(1, "exactly one notification per item, then silence");
demoted.Count(x => x.Handle == 2).ShouldBe(1);
demoted.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
rig.Sink.WriteValue(raw, 4.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
var promoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
promoted.ShouldContain(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0));
promoted.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0),
"the promotion value is queued ignoring the deadband");
}
// ------------------------------------------------------------------ T7 first-delivery decision record (D4)
/// <summary>Timing 1 (T0): Good samples queued in the item while Active, a client-handle change, then demotion.</summary>
[Fact]
public async Task Timing1_queued_good_samples_then_demotion_first_delivery_is_projected_and_republish_matches()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717T1", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("q", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.MaterialiseScripted();
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
var tagItem = await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct);
var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
await AddItemAsync(client, sub, severity, 3, 1, ct);
(await PublishDataAsync(client, ct)).NotificationMessage.NotificationData.Count.ShouldBeGreaterThan(0);
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
// Non-ordinary changes queued in the same window: i=2267 and a condition child (Severity).
new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
Microsoft.Extensions.Logging.Abstractions.NullLogger<SdkServiceLevelPublisher>.Instance).Publish(90);
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 650, Message: "child"));
await client.ModifyMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
{
new MonitoredItemModifyRequest { MonitoredItemId = tagItem,
RequestedParameters = new MonitoringParameters { ClientHandle = 99, SamplingInterval = 0, QueueSize = 10, DiscardOldest = true } },
}, ct);
DemoteDeliveryOnly(projection);
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
var msg = await PublishDataAsync(client, ct);
var changes = DataChanges(msg.NotificationMessage);
changes.Where(x => x.Handle is 1 or 99).ShouldNotBeEmpty();
changes.Where(x => x.Handle is 1 or 99).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService && x.Value.WrappedValue.IsNull,
"Good samples queued before demotion are first-delivered projected (D4)");
changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not an ordinary tag");
changes.ShouldContain(x => x.Handle == 3 && x.Value.StatusCode == StatusCodes.Good, "a condition child is not an ordinary tag");
var again = await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct);
DataChanges(again.NotificationMessage).Where(x => x.Handle is 1 or 99)
.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService, "Republish returns the recorded projected choice");
}
/// <summary>Timing 2 (T0): messages prebuilt while Active (one notification per message), then demotion.</summary>
[Fact]
public async Task Timing2_prebuilt_messages_then_demotion_next_publish_is_projected_and_unsent_republish_is_unavailable()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717T2", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("p", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 1, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
DataChanges((await PublishDataAsync(client, ct)).NotificationMessage).Count.ShouldBe(1);
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
rig.Sink.WriteValue(raw, 20.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 21.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 22.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
var a = await PublishDataAsync(client, ct); // builds several messages while Active; the first is delivered Good
DataChanges(a.NotificationMessage).Single().Value.StatusCode.ShouldBe(StatusCodes.Good);
DemoteDeliveryOnly(projection);
var next = a.NotificationMessage.SequenceNumber + 1;
(await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, next, ct).AsTask())).StatusCode
.ShouldBe(StatusCodes.BadMessageNotAvailable, "2.0.1: a prebuilt, never-sent message cannot be first-delivered by Republish");
var b = await PublishDataAsync(client, ct);
b.NotificationMessage.SequenceNumber.ShouldBe(next);
DataChanges(b.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService,
"a message built while Active but first delivered after demotion is projected");
var bRe = await client.RepublishAsync(H(), sub, next, ct);
DataChanges(bRe.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
}
/// <summary>Timing 3 (T0): Good delivered while Active, then demotion — Republish returns the original Good; after an
/// acknowledge it is gone.</summary>
[Fact]
public async Task Timing3_good_delivered_while_active_republishes_good_after_demotion_and_ack_drops_it()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717T3", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("d", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
var first = await PublishDataAsync(client, ct);
DataChanges(first.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
Demote(projection);
var again = await client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct);
DataChanges(again.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good,
"Republish of a message first delivered while Active returns the original (D4 boundary)");
await client.PublishAsync(H(), new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = first.NotificationMessage.SequenceNumber } }, ct);
(await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct).AsTask()))
.StatusCode.ShouldBe(StatusCodes.BadMessageNotAvailable);
}
/// <summary>The D4 boundary alone (spec §4.7 demotion step 2), without the projection that follows it, so the
/// decision record is what the test observes.</summary>
private static void DemoteDeliveryOnly(IStandbyProjection p) => p.SetStandbyDelivery(true);
// ------------------------------------------------------------------ raw client helpers
private static async Task<uint> AddItemAsync(SessionClient client, uint sub, NodeId node, uint handle, uint queue, CancellationToken ct,
MonitoringFilter? filter = null)
{
var r = await client.CreateMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
{
new MonitoredItemCreateRequest
{
ItemToMonitor = new ReadValueId { NodeId = node, AttributeId = Attributes.Value },
MonitoringMode = MonitoringMode.Reporting,
RequestedParameters = new MonitoringParameters
{
ClientHandle = handle, SamplingInterval = 0, QueueSize = queue, DiscardOldest = true,
Filter = filter is null ? ExtensionObject.Null : new ExtensionObject(filter),
},
},
}, ct);
StatusCode.IsGood(r.Results[0].StatusCode).ShouldBeTrue($"create {node}: {r.Results[0].StatusCode}");
return r.Results[0].MonitoredItemId;
}
internal static List<(uint Handle, DataValue Value)> DataChanges(NotificationMessage m)
{
var list = new List<(uint, DataValue)>();
for (var i = 0; i < m.NotificationData.Count; i++)
{
if (m.NotificationData[i].TryGetValue<DataChangeNotification>(out var d))
for (var j = 0; j < d.MonitoredItems.Count; j++) list.Add((d.MonitoredItems[j].ClientHandle, d.MonitoredItems[j].Value));
}
return list;
}
private static async Task<PublishResponse> PublishDataAsync(SessionClient client, CancellationToken ct)
{
for (var i = 0; i < 40; i++)
{
var r = await client.PublishAsync(H(), default, ct);
if (r.NotificationMessage.NotificationData.Count > 0) return r;
}
throw new InvalidOperationException("no data message");
}
private static async Task<List<(uint Handle, DataValue Value)>> CollectAsync(SessionClient client, CancellationToken ct, TimeSpan window)
{
var collected = new List<(uint, DataValue)>();
var until = DateTime.UtcNow + window;
while (DateTime.UtcNow < until)
{
var r = await client.PublishAsync(H(), default, ct);
collected.AddRange(DataChanges(r.NotificationMessage));
}
return collected;
}
}
@@ -0,0 +1,250 @@
using Opc.Ua;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
/// <summary>
/// Gitea #717 spec §4.6 (the T0-adopted mechanism) at unit level: the ordinary-tag marker and its validation, the
/// Standby rewrite, the (SubscriptionId, SequenceNumber) decision record and its bounds, and the redundancy option
/// rules of spec §3.
/// </summary>
public sealed class WarmStandbyUnitTests
{
private static DataValue Good(double v, int secondsAgo = 0) =>
new(new Variant(v), StatusCodes.Good, (DateTimeUtc)DateTime.UtcNow.AddSeconds(-secondsAgo), DateTimeUtc.Now);
private static MonitoredItemNotification Marked(uint handle, DataValue value)
{
var n = new MonitoredItemNotification { ClientHandle = handle, Value = value };
n.Message = new OrdinaryTagMarker(handle, value);
return n;
}
private static NotificationMessage Message(uint seq, params MonitoredItemNotification[] items) => new()
{
SequenceNumber = seq,
PublishTime = DateTimeUtc.Now,
NotificationData = new List<ExtensionObject>
{
new(new DataChangeNotification { MonitoredItems = items.ToList() }),
},
};
private static List<MonitoredItemNotification> Items(NotificationMessage m)
{
m.NotificationData[0].TryGetValue<DataChangeNotification>(out var d).ShouldBeTrue();
return Enumerable.Range(0, d!.MonitoredItems.Count).Select(i => d.MonitoredItems[i]).ToList();
}
// ------------------------------------------------------------------ marker + rewrite
[Fact]
public void A_marked_ordinary_tag_notification_is_rewritten_and_the_input_is_not_mutated()
{
var value = Good(5);
var tag = Marked(1, value);
var other = new MonitoredItemNotification { ClientHandle = 2, Value = Good(6) };
var input = Message(7, tag, other);
var projected = StandbyNotificationRewriter.Project(input, out var rewritten);
rewritten.ShouldBe(1);
ReferenceEquals(projected, input).ShouldBeFalse();
var items = Items(projected);
items[0].ClientHandle.ShouldBe(1u);
items[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService);
items[0].Value.WrappedValue.IsNull.ShouldBeTrue();
items[0].Value.ServerTimestamp.ShouldBe(value.ServerTimestamp);
ReferenceEquals(items[1], other).ShouldBeTrue("a non-ordinary notification is passed through untouched");
projected.SequenceNumber.ShouldBe(7u);
// The SDK-owned input (the per-delivery clone) is untouched.
Items(input)[0].Value.StatusCode.ShouldBe(StatusCodes.Good);
ReferenceEquals(Items(input)[0], tag).ShouldBeTrue();
}
[Fact]
public void A_stale_marker_from_a_recycled_payload_is_ignored()
{
// 2.0.1 pools MonitoredItemNotification payloads and Initialize() resets only ClientHandle and Value, so a
// recycled object can carry a marker made for another notification. Handle or value mismatch ⇒ not ordinary.
var otherHandle = new MonitoredItemNotification { ClientHandle = 3, Value = Good(5) };
otherHandle.Message = new OrdinaryTagMarker(1, otherHandle.Value);
var otherValue = new MonitoredItemNotification { ClientHandle = 1, Value = Good(6) };
otherValue.Message = new OrdinaryTagMarker(1, Good(5, secondsAgo: 10));
OrdinaryTagMarker.IsOrdinaryTag(otherHandle).ShouldBeFalse();
OrdinaryTagMarker.IsOrdinaryTag(otherValue).ShouldBeFalse();
var input = Message(1, otherHandle, otherValue);
ReferenceEquals(StandbyNotificationRewriter.Project(input, out var rewritten), input).ShouldBeTrue();
rewritten.ShouldBe(0);
}
[Fact]
public void Bad_and_unmarked_notifications_are_never_rewritten()
{
var bad = Marked(1, new DataValue(Variant.Null, StatusCodes.BadNoCommunication, DateTimeUtc.Now, DateTimeUtc.Now));
var unmarked = new MonitoredItemNotification { ClientHandle = 2, Value = Good(1) };
var input = Message(1, bad, unmarked);
ReferenceEquals(StandbyNotificationRewriter.Project(input, out _), input).ShouldBeTrue();
}
[Fact]
public void Uncertain_ordinary_tag_values_are_projected_too()
{
var uncertain = Marked(1, new DataValue(new Variant(1.0), StatusCodes.UncertainLastUsableValue, DateTimeUtc.Now, DateTimeUtc.Now));
Items(StandbyNotificationRewriter.Project(Message(1, uncertain), out _))[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService);
}
// ------------------------------------------------------------------ SDK-field contract (regression pin)
/// <summary>
/// The mechanism stands on two SDK 2.0.1 behaviours of <see cref="MonitoredItemNotification.Message"/>: the SDK's
/// per-delivery <c>CoreUtils.Clone</c> COPIES it, and the binary encoder NEVER writes it. An SDK upgrade that
/// changes either breaks D4 silently (no marker ⇒ no rewrite) or leaks a server object onto the wire — this
/// fails loudly instead.
/// </summary>
[Fact]
public void Sdk_contract_the_marker_survives_the_delivery_clone_and_is_never_encoded()
{
var value = Good(5);
var marked = Message(9, Marked(1, value));
var clone = CoreUtils.Clone(marked);
ReferenceEquals(clone, marked).ShouldBeFalse();
var cloned = Items(clone!)[0];
ReferenceEquals(cloned, Items(marked)[0]).ShouldBeFalse("the clone is deep");
cloned.Message.ShouldBeOfType<OrdinaryTagMarker>("the SDK clone must copy MonitoredItemNotification.Message");
OrdinaryTagMarker.IsOrdinaryTag(cloned).ShouldBeTrue("the copied marker still validates against the clone");
var context = ServiceMessageContext.Create(DefaultTelemetry.Create(_ => { }));
var unmarked = Message(9, new MonitoredItemNotification { ClientHandle = 1, Value = value });
unmarked.PublishTime = marked.PublishTime;
var withMarker = BinaryEncoder.EncodeMessage(marked, context);
var without = BinaryEncoder.EncodeMessage(unmarked, context);
withMarker.ShouldBe(without, "the helper field must never reach the wire");
var decoded = BinaryDecoder.DecodeMessage<NotificationMessage>(withMarker, context);
Items(decoded!)[0].Message.ShouldBeNull();
}
// ------------------------------------------------------------------ decision record
[Fact]
public void The_first_delivery_records_the_decision_and_every_later_delivery_reuses_it()
{
var record = new FirstDeliveryRecord(standby: false);
record.Decide(1, 10).ShouldBeFalse();
record.SetStandby(true);
record.Decide(1, 10).ShouldBeFalse("a Republish of a message first delivered Active stays Active (D4)");
record.Decide(1, 11).ShouldBeTrue("a first delivery after the flip is Standby");
record.SetStandby(false);
record.Decide(1, 11).ShouldBeTrue("a Republish of a message first delivered Standby stays projected");
record.Decide(2, 11).ShouldBeFalse("keyed by subscription too");
}
[Fact]
public void The_record_is_pruned_by_available_sequence_numbers_and_bounded()
{
var record = new FirstDeliveryRecord(standby: true);
for (uint s = 1; s <= 5; s++) record.Decide(1, s);
record.Prune(1, [3, 4], justDelivered: 5);
record.Count.ShouldBe(3);
for (uint s = 100; s < 100 + FirstDeliveryRecord.MaxPerSubscription + 50; s++) record.Decide(1, s);
record.Count.ShouldBe(FirstDeliveryRecord.MaxPerSubscription);
record.Decide(2, 1);
record.SweepDead(new HashSet<uint> { 2 });
record.SubscriptionCount.ShouldBe(1);
}
[Fact]
public void Dead_subscriptions_are_evicted_least_recently_used_beyond_the_bound()
{
var record = new FirstDeliveryRecord(standby: false);
for (uint sub = 1; sub <= FirstDeliveryRecord.MaxSubscriptions; sub++) record.Decide(sub, 1);
record.Decide(1, 2); // touch subscription 1 so 2 is the least recently used
record.Decide(FirstDeliveryRecord.MaxSubscriptions + 1, 1);
record.SubscriptionCount.ShouldBe(FirstDeliveryRecord.MaxSubscriptions);
}
/// <summary>
/// A role flip and a first delivery race (spec §4.6 item 2): the flip waits for a decision in progress, so the
/// decision is ordered strictly before or after it — never torn. Driven by a deterministic barrier, not sleeps.
/// </summary>
[Fact]
public async Task A_flip_waits_for_a_decision_in_progress_and_orders_strictly_against_it()
{
for (var round = 0; round < 50; round++)
{
var record = new FirstDeliveryRecord(standby: false);
using var inside = new ManualResetEventSlim();
using var release = new ManualResetEventSlim();
record.DecisionBarrier = () => { inside.Set(); release.Wait(TimeSpan.FromSeconds(10)); };
var ct = TestContext.Current.CancellationToken;
var decision = Task.Run(() => record.Decide(1, (uint)round + 1), ct);
inside.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue();
var flip = Task.Run(() => record.SetStandby(true), ct);
(await Task.WhenAny(flip, Task.Delay(100, ct))).ShouldNotBe(flip, "the flip must wait for the decision in progress");
release.Set();
(await decision).ShouldBeFalse("the decision read the state before the flip");
await flip;
record.DecisionBarrier = null;
record.Decide(1, (uint)round + 1).ShouldBeFalse("and recorded it");
record.Decide(1, (uint)round + 1000).ShouldBeTrue("a later first delivery sees the flip");
}
}
// ------------------------------------------------------------------ WarmStandbyState
[Fact]
public void The_Off_state_never_projects_suppresses_or_refuses()
{
var off = new WarmStandbyState(false);
off.DataProjected = true;
off.AlarmsSuppressed = true;
off.DataProjected.ShouldBeFalse();
off.AlarmsSuppressed.ShouldBeFalse();
off.RefusesMutations.ShouldBeFalse();
off.Delivery.ShouldBeNull();
var warm = new WarmStandbyState(true);
warm.DataProjected.ShouldBeTrue("a Warm node boots Standby");
warm.AlarmsSuppressed.ShouldBeTrue();
warm.RefusesMutations.ShouldBeTrue();
warm.Delivery!.Standby.ShouldBeTrue();
}
// ------------------------------------------------------------------ OpcUa:Redundancy rules (spec §3)
[Theory]
[InlineData("Unset", new string[0], true)]
[InlineData("", new string[0], true)]
[InlineData("none", new string[0], true)]
[InlineData("None", new[] { "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:peer" }, true)]
[InlineData("WARM", new[] { "urn:peer", "urn:peer2" }, true)]
[InlineData("Warm", new string[0], false)]
[InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:self" }, false)]
[InlineData("Hot", new[] { "urn:peer" }, false)]
[InlineData("2", new[] { "urn:peer" }, false)]
[InlineData("Transparent", new[] { "urn:peer" }, false)]
public void Redundancy_option_rules(string mode, string[] peers, bool valid)
{
var options = new OpcUaRedundancyOptions { Mode = mode };
options.Check("urn:self", peers).Count.ShouldBe(valid ? 0 : 1, string.Join("; ", options.Check("urn:self", peers)));
}
[Fact]
public void Redundancy_defaults_are_Unset_with_promote_on()
{
var options = new OpcUaApplicationHostOptions();
options.Redundancy.ParsedMode.ShouldBe(OpcUaRedundancyMode.Unset);
options.Redundancy.PromoteWhilePrimaryUnreachable.ShouldBeTrue();
options.Redundancy.ToSettings().ShouldBe(WarmRedundancySettings.Unset);
}
}
@@ -0,0 +1,220 @@
using System.Collections.Concurrent;
using System.Text;
using Akka.Actor;
using Akka.TestKit;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
/// <summary>
/// Gitea #717 spec §4.1–§4.3 and §4.7 in <see cref="OpcUaPublishActor"/>: the Warm verdict from membership + the
/// accepted revisioned snapshot, the bands, and the transition step order — with a recording projection and
/// ServiceLevel publisher writing into ONE ordered log, so the invariant "an Active-band level is never visible over
/// projected data" is checked on the actual sequence.
/// </summary>
public sealed class OpcUaPublishActorWarmTests : RuntimeActorTestBase
{
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
private readonly ConcurrentQueue<string> _log = new();
private long _seq;
private NodeId Self
{
get
{
var cluster = Akka.Cluster.Cluster.Get(Sys);
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
}
}
private IActorRef Start(WarmRedundancySettings redundancy, LoggingPublisher? publisher = null) =>
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher ?? new LoggingPublisher(_log), localNode: Self, dbLess: true,
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: redundancy,
standbyProjection: new RecordingProjection(_log)));
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
new(new[]
{
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Self, DateTime.UtcNow),
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Other, DateTime.UtcNow),
},
CorrelationId.NewId(),
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
private string[] Steps => _log.ToArray();
private void AwaitLogEndsWith(params string[] tail) =>
AwaitAssert(() => Steps.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", Steps)), PresenceBudget,
cancellationToken: TestContext.Current.CancellationToken);
private static readonly string[] Promotion = ["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL:250"];
private static readonly string[] Demotion = ["SL:90", "delivery:True", "alarms:True", "proj:True", "queue:True"];
[Fact]
public void Promotion_and_demotion_run_in_the_spec_order()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
Steps.ShouldBe(Promotion, "boot is Standby; the first accepted snapshot promotes and nothing else ran");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith(Demotion);
AssertNoActiveLevelOverProjectedData();
}
/// <summary>Spec §4.7 invariant over the whole sequence: every published level ≥ 100 comes after the projection was
/// last opened, never while it is closed.</summary>
private void AssertNoActiveLevelOverProjectedData()
{
var projected = true; // a Warm node boots projected
foreach (var step in Steps)
{
if (step == "proj:True") projected = true;
else if (step == "proj:False") projected = false;
else if (step.StartsWith("SL:", StringComparison.Ordinal) && byte.Parse(step[3..]) >= 100)
projected.ShouldBeFalse($"an Active-band level was visible over projected data: {string.Join(" ", Steps)}");
}
}
[Fact]
public void An_unrevisioned_snapshot_never_makes_a_Warm_node_Active()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self, revisioned: false));
AwaitLogEndsWith("SL:90");
Steps.ShouldBe(new[] { "SL:90" });
}
/// <summary>Spec §4.2: after a newer snapshot names this node Primary, an OLDER incarnation's late publish naming the
/// peer is dropped — the node stays Active. Then a genuinely newer one demotes it (positive control).</summary>
[Fact]
public void An_older_revision_after_a_newer_one_is_dropped()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self, upNumber: 5));
AwaitLogEndsWith(Promotion);
actor.Tell(Snapshot(Other, upNumber: 4)); // a departed older singleton's late heartbeat
actor.Tell(Snapshot(Self, upNumber: 5)); // a fence: processed after the stale one
AwaitAssert(() => Steps.Count(s => s == "SL:250").ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
Steps.ShouldNotContain("delivery:True", "the stale snapshot must not demote");
actor.Tell(Snapshot(Other, upNumber: 6));
AwaitLogEndsWith(Demotion);
}
/// <summary>Spec §4.7: if the Active level cannot be published, the promotion stops before it and retries; the
/// Standby level stays visible meanwhile.</summary>
[Fact]
public void A_failed_Active_level_publish_halts_before_it_and_retries()
{
var publisher = new LoggingPublisher(_log) { FailNext = 1, FailLevel = 250 };
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
actor.Tell(Snapshot(Self));
AwaitAssert(() => Steps.ShouldContain("SL:250"), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
var log = Steps;
log.Take(6).ShouldBe(["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL-FAILED:250"]);
log.Last().ShouldBe("SL:250", "retried without a new snapshot");
}
/// <summary>A demotion continues when the Standby level publish fails: failing safe degrades the data.</summary>
[Fact]
public void A_failed_Standby_level_publish_still_degrades_the_data()
{
var publisher = new LoggingPublisher(_log);
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
publisher.FailNext = 1;
publisher.FailLevel = 90;
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL-FAILED:90", "delivery:True", "alarms:True", "proj:True", "queue:True");
}
[Fact]
public void A_burst_of_alternating_verdicts_converges_on_the_last()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
for (var i = 0; i < 20; i++) actor.Tell(Snapshot(i % 2 == 0 ? Self : Other));
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
AssertNoActiveLevelOverProjectedData();
}
/// <summary>The default-off pin at the actor: Unset never touches the projection and keeps today's levels.</summary>
[Fact]
public void Unset_never_drives_the_projection_and_keeps_todays_levels()
{
var actor = Start(WarmRedundancySettings.Unset);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL:240");
Steps.ShouldBe(new[] { "SL:250", "SL:240" });
}
[Fact]
public void A_deployed_cluster_mode_that_disagrees_with_the_local_mode_is_logged_and_counted()
{
var artifact = Encoding.UTF8.GetBytes(
"{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":\"None\"}],\"Nodes\":[]}");
DeploymentArtifact.ReadClusterRedundancyMode(artifact, "x:1").ShouldBe("None");
DeploymentArtifact.ReadClusterRedundancyMode(
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":1}]}"), "x:1").ShouldBe("Warm");
DeploymentArtifact.ReadClusterRedundancyMode(
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"a\",\"RedundancyMode\":\"None\"},{\"ClusterId\":\"b\",\"RedundancyMode\":\"Hot\"}],"
+ "\"Nodes\":[{\"NodeId\":\"x:1\",\"ClusterId\":\"b\"}]}"), "x:1").ShouldBe("Hot");
using var listener = new System.Diagnostics.Metrics.MeterListener();
long mismatches = 0;
listener.InstrumentPublished = (instrument, l) =>
{
if (ReferenceEquals(instrument, ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.RedundancyModeMismatch))
l.EnableMeasurementEvents(instrument);
};
listener.SetMeasurementEventCallback<long>((_, value, _, _) => Interlocked.Add(ref mismatches, value));
listener.Start();
var warmActor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
localNode: NodeId.Parse("x:1"), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm)));
EventFilter.Error(contains: "RedundancyMode for this node's cluster is None").ExpectOne(() =>
warmActor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), Artifact: artifact)),
TestContext.Current.CancellationToken);
Interlocked.Read(ref mismatches).ShouldBeGreaterThanOrEqualTo(1);
}
}
/// <summary>Writes each published level into the shared log; can be told to fail the next N publishes of a level.</summary>
internal sealed class LoggingPublisher(ConcurrentQueue<string> log) : IServiceLevelPublisher
{
public int FailNext { get; set; }
public byte FailLevel { get; set; }
public void Publish(byte serviceLevel)
{
if (FailNext > 0 && serviceLevel == FailLevel)
{
FailNext--;
log.Enqueue($"SL-FAILED:{serviceLevel}");
throw new InvalidOperationException("injected ServiceLevel publish failure");
}
log.Enqueue($"SL:{serviceLevel}");
}
}
@@ -0,0 +1,91 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
/// <summary>Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7).</summary>
public sealed class WarmRoleAuthorityTests
{
private static readonly NodeId Self = NodeId.Parse("a:1");
private static readonly NodeId Peer = NodeId.Parse("b:1");
private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true,
NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) =>
new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote);
[Fact]
public void Active_iff_membership_and_the_accepted_snapshot_both_name_self()
{
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf)
{
var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self));
verdict.Role.ShouldBe(WarmRole.Standby);
verdict.Disagreement.ShouldBeTrue();
}
[Fact]
public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership()
{
var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null));
verdict.Role.ShouldBe(WarmRole.Standby);
verdict.Reason.ShouldBe("boot");
}
[Fact]
public void Not_up_or_endpoint_failed_is_Standby()
{
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby);
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby);
}
/// <summary>Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes.</summary>
[Fact]
public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option()
{
var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true);
WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby,
"a reachable Primary is never overridden");
WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role
.ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby");
}
[Fact]
public void A_late_bind_replays_the_retained_state_but_not_the_actions()
{
var deferred = new DeferredStandbyProjection();
deferred.SetDataProjection(false);
deferred.SetStandbyDelivery(false);
deferred.SetAlarmSuppression(false);
deferred.QueueCurrentToLiveItems(false).ShouldBe(0);
deferred.RefreshAllEventItems().ShouldBe(0);
var inner = new RecordingProjection();
deferred.SetInner(inner);
inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" });
deferred.QueueCurrentToLiveItems(true);
inner.Log.Last().ShouldBe("queue:True");
}
}
/// <summary>Records every <see cref="IStandbyProjection"/> step into a shared, ordered log.</summary>
internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue<string>? log = null) : IStandbyProjection
{
public System.Collections.Concurrent.ConcurrentQueue<string> Log { get; } = log ?? new();
public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}");
public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}");
public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}");
public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; }
public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; }
}