feat(717): Warm non-transparent redundancy, default-off (T1-T10)

OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start).
- T1 options + validator + effective-URI check + boot banner; Warm needs the driver role.
- T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only
  ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None
  and the SDK's orphan i=11314 deleted; Unset untouched.
- T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older
  revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active.
- T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable.
- T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec
  4.7 order (Active level last and retried; Standby level first, data degraded regardless).
- T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState.
- T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by
  (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's
  cached copy is never mutated.
- T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters.
- T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion
  refresh via ISubscription.ConditionRefresh2Async on every live event item.
- T10 deploy-time RedundancyMode cross-check + six instruments.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:26:36 -05:00
parent dcff5ae994
commit 7829b937d1
27 changed files with 3056 additions and 29 deletions
@@ -62,4 +62,44 @@ public static class ServiceLevelCalculator
return (byte)Math.Clamp(basis + (h.IsDriverPrimary ? 10 : 0), 0, 255);
}
/// <summary>Warm Standby, healthy (Gitea #717 spec §4.3): Degraded band.</summary>
public const byte WarmStandbyHealthy = 90;
/// <summary>Warm Standby, stale or DB unreachable (spec §4.3): Degraded band.</summary>
public const byte WarmStandbyDegraded = 50;
/// <summary>Warm Active, critically degraded (spec §4.3): the lowest level a data-serving Active node publishes.</summary>
public const byte WarmActiveCritical = 100;
/// <summary>Warm Active, stale config with the DB reachable (spec §4.3).</summary>
public const byte WarmActiveStale = 200;
/// <summary>Warm Active, healthy (spec §4.3): 240 + the 10-point Primary bonus.</summary>
public const byte WarmActiveHealthy = 250;
/// <summary>
/// The Warm ServiceLevel bands (Gitea #717 spec §4.3). <see cref="Compute"/> is unchanged and still serves
/// <c>Unset</c> and <c>None</c>; this is used only under <c>OpcUa:Redundancy:Mode=Warm</c>.
/// </summary>
/// <remarks>
/// Invariant: every Standby level (≤ <see cref="WarmStandbyHealthy"/>) is below every level a data-serving
/// Active node publishes (≥ <see cref="WarmActiveCritical"/>), so a client that picks the highest
/// ServiceLevel never prefers a Standby. A member that is not Up/Joining, or whose own endpoint failed, is 0
/// whatever its role (unchanged from <see cref="Compute"/>, lmxopcua#762).
/// </remarks>
/// <param name="h">The node's health facts; <see cref="NodeHealthInputs.IsDriverPrimary"/> is ignored — the
/// Warm verdict <paramref name="isActive"/> replaces it.</param>
/// <param name="isActive">The Warm role verdict (spec §4.1).</param>
/// <returns>The ServiceLevel byte.</returns>
public static byte ComputeWarm(NodeHealthInputs h, bool isActive)
{
if (h.MemberState is not (MemberStatus.Up or MemberStatus.Joining)) return 0;
if (!h.EndpointListening) return 0;
var healthy = h.DbReachable && h.OpcUaProbeOk && !h.Stale;
if (!isActive) return healthy ? WarmStandbyHealthy : WarmStandbyDegraded;
if (healthy) return WarmActiveHealthy;
return h.DbReachable && h.Stale ? WarmActiveStale : WarmActiveCritical;
}
}
@@ -6,6 +6,74 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
/// Broadcast whenever the cluster's redundancy topology changes (node up/down, role-leader
/// change, partition heal). Subscribers compute their local OPC UA ServiceLevel from this.
/// </summary>
/// <param name="Nodes">One entry per cluster member.</param>
/// <param name="CorrelationId">A random id per publish (diagnostics only; it carries no order).</param>
/// <param name="Revision">The publisher's ordered revision (Gitea #717, spec §4.2). <c>null</c> from a pre-#717
/// publisher during a rolling upgrade: such a snapshot still feeds the existing gates, but can never make a Warm
/// node Active.</param>
public sealed record RedundancyStateChanged(
IReadOnlyList<NodeRedundancyState> Nodes,
CorrelationId CorrelationId);
CorrelationId CorrelationId,
SnapshotRevision? Revision = null);
/// <summary>
/// The order of <see cref="RedundancyStateChanged"/> snapshots (Gitea #717, spec §4.2), compared
/// lexicographically. Within one cluster incarnation the singleton always lives on the oldest member, and every
/// surviving member has a larger up-number than a departed oldest one, so <see cref="PublisherUpNumber"/> is
/// monotonic across handovers; <see cref="IncarnationTick"/> orders a singleton restarted in the same process;
/// <see cref="Sequence"/> orders the publishes of one incarnation.
/// </summary>
/// <param name="PublisherUpNumber">The <c>Member.UpNumber</c> of the node hosting the singleton.</param>
/// <param name="IncarnationTick"><c>Environment.TickCount64</c> at the singleton's <c>PreStart</c>.</param>
/// <param name="Sequence">Incremented on every publish, heartbeats included.</param>
public sealed record SnapshotRevision(long PublisherUpNumber, long IncarnationTick, long Sequence)
: IComparable<SnapshotRevision>
{
/// <inheritdoc />
public int CompareTo(SnapshotRevision? other)
{
if (other is null) return 1;
var c = PublisherUpNumber.CompareTo(other.PublisherUpNumber);
if (c != 0) return c;
c = IncarnationTick.CompareTo(other.IncarnationTick);
return c != 0 ? c : Sequence.CompareTo(other.Sequence);
}
/// <inheritdoc />
public override string ToString() => $"{PublisherUpNumber}.{IncarnationTick}.{Sequence}";
}
/// <summary>Why a snapshot was not accepted by a <see cref="SnapshotRevisionGate"/>.</summary>
public enum SnapshotRejection
{
/// <summary>Accepted.</summary>
None,
/// <summary>Its revision is not greater than the last accepted one (a late publish of an older incarnation).</summary>
OlderRevision,
/// <summary>It carries no revision, while revisioned snapshots are being ordered.</summary>
Unrevisioned,
}
/// <summary>
/// The subscriber-side order rule of spec §4.2: accept a snapshot only if its revision is greater than the last
/// accepted one. Not thread-safe; owned by one actor.
/// </summary>
public sealed class SnapshotRevisionGate
{
/// <summary>The last accepted revision; <c>null</c> until a revisioned snapshot was accepted.</summary>
public SnapshotRevision? LastAccepted { get; private set; }
/// <summary>Decide one snapshot. An unrevisioned snapshot is reported <see cref="SnapshotRejection.Unrevisioned"/>
/// and never advances the order (the caller decides whether the existing gates still use it).</summary>
/// <param name="msg">The snapshot.</param>
/// <returns><see cref="SnapshotRejection.None"/> when accepted.</returns>
public SnapshotRejection Offer(RedundancyStateChanged msg)
{
if (msg.Revision is not { } revision) return SnapshotRejection.Unrevisioned;
if (LastAccepted is { } last && revision.CompareTo(last) <= 0) return SnapshotRejection.OlderRevision;
LastAccepted = revision;
return SnapshotRejection.None;
}
}
@@ -190,6 +190,44 @@ public static class OtOpcUaTelemetry
Meter.CreateCounter<long>("otopcua.opcua.condition_events.superseded", unit: "{delivery}",
description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state.");
// ---------------- Warm redundancy (Gitea #717, spec §7) ----------------
private static int _warmRole = -1;
/// <summary>Records this process's Warm role for <see cref="WarmRole"/>: 1 Active, 0 Standby, −1 not Warm.</summary>
/// <param name="value">The role value.</param>
public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value);
/// <summary>The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm.</summary>
public static readonly ObservableGauge<int> WarmRole =
Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}",
description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm.");
/// <summary>Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed).</summary>
public static readonly Counter<long> WarmTransition =
Meter.CreateCounter<long>("otopcua.redundancy.warm_transition", unit: "{transition}",
description: "Warm role transitions (direction=promote|demote, reason).");
/// <summary>Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned).</summary>
public static readonly Counter<long> SnapshotRejected =
Meter.CreateCounter<long>("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}",
description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned).");
/// <summary>The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.</summary>
public static readonly Counter<long> RedundancyModeMismatch =
Meter.CreateCounter<long>("otopcua.redundancy.mode_mismatch", unit: "{apply}",
description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.");
/// <summary>Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4).</summary>
public static readonly Counter<long> StandbyProjectedNotifications =
Meter.CreateCounter<long>("otopcua.opcua.standby.projected_notifications", unit: "{notification}",
description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery.");
/// <summary>New alarm condition events not delivered on a Warm Standby (still stamped).</summary>
public static readonly Counter<long> StandbySuppressedAlarmEvents =
Meter.CreateCounter<long>("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}",
description: "New alarm condition events suppressed on a Warm standby.");
// ---------------- Convenience helpers ----------------
/// <summary>
@@ -0,0 +1,112 @@
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
/// <summary>
/// The data-plane side of a Warm role transition (Gitea #717, spec §4.7). <c>OpcUaPublishActor</c> owns the
/// verdict and drives these steps in a fixed order; the node manager implements them. Every member is
/// non-blocking and idempotent. Under <see cref="OpcUaRedundancyMode.Unset"/> and
/// <see cref="OpcUaRedundancyMode.None"/> nothing calls it.
/// </summary>
public interface IStandbyProjection
{
/// <summary>Opens (<c>false</c>) or closes (<c>true</c>) the per-tag read projection: while projected, every
/// ordinary-tag <c>Value</c> read and sample answers <c>BadOutOfService</c>/null, and writes and Part 9 condition
/// methods are refused <c>BadInvalidState</c>. The stored value keeps updating from acquisition.</summary>
/// <param name="projected">Whether ordinary-tag values are projected.</param>
void SetDataProjection(bool projected);
/// <summary>Switches the first-delivery decision (spec §4.6, D4): a Publish response delivered for the first time
/// while <paramref name="standby"/> is recorded as Standby and its ordinary-tag notifications are rewritten.</summary>
/// <param name="standby">Whether first deliveries are decided Standby.</param>
void SetStandbyDelivery(bool standby);
/// <summary>Suppresses (<c>true</c>) or restores delivery of NEW alarm condition events (spec §4.5). Condition
/// state keeps being stamped, so a later refresh carries current, ackable state.</summary>
/// <param name="suppressed">Whether new alarm events are suppressed.</param>
void SetAlarmSuppression(bool suppressed);
/// <summary>Queues each live, non-stood-down ordinary-tag data item one value, ignoring its filters: the current
/// stored value (promotion) or <c>BadOutOfService</c>/null (demotion).</summary>
/// <param name="projected">Whether to queue the projected value instead of the stored one.</param>
/// <returns>The number of items queued to.</returns>
int QueueCurrentToLiveItems(bool projected);
/// <summary>Runs a ConditionRefresh on every live event item (Server-object, folder and direct), through the
/// existing refresh paths and their ACL decisions (spec §4.5 promotion refresh).</summary>
/// <returns>The number of event items refreshed.</returns>
int RefreshAllEventItems();
}
/// <summary>
/// Late-binding <see cref="IStandbyProjection"/> (the <see cref="DeferredServiceLevelPublisher"/> pattern): the
/// actor gets it at construction, the node manager only exists after SDK start. The latest requested
/// projection / delivery / suppression state is retained and replayed on bind (the #719 lesson); the two
/// queue/refresh actions are not replayed — before bind there are no items to act on.
/// </summary>
public sealed class DeferredStandbyProjection : IStandbyProjection
{
private readonly object _gate = new();
private IStandbyProjection? _inner;
private bool? _projected;
private bool? _standby;
private bool? _suppressed;
/// <summary>Bind (and replay the retained state), or unbind with <c>null</c>.</summary>
/// <param name="inner">The live implementation, or null to unbind.</param>
public void SetInner(IStandbyProjection? inner)
{
lock (_gate)
{
_inner = inner;
if (inner is null) return;
if (_projected is { } p) inner.SetDataProjection(p);
if (_standby is { } s) inner.SetStandbyDelivery(s);
if (_suppressed is { } a) inner.SetAlarmSuppression(a);
}
}
/// <inheritdoc />
public void SetDataProjection(bool projected)
{
lock (_gate)
{
_projected = projected;
_inner?.SetDataProjection(projected);
}
}
/// <inheritdoc />
public void SetStandbyDelivery(bool standby)
{
lock (_gate)
{
_standby = standby;
_inner?.SetStandbyDelivery(standby);
}
}
/// <inheritdoc />
public void SetAlarmSuppression(bool suppressed)
{
lock (_gate)
{
_suppressed = suppressed;
_inner?.SetAlarmSuppression(suppressed);
}
}
/// <inheritdoc />
public int QueueCurrentToLiveItems(bool projected)
{
IStandbyProjection? inner;
lock (_gate) inner = _inner;
return inner?.QueueCurrentToLiveItems(projected) ?? 0;
}
/// <inheritdoc />
public int RefreshAllEventItems()
{
IStandbyProjection? inner;
lock (_gate) inner = _inner;
return inner?.RefreshAllEventItems() ?? 0;
}
}
@@ -0,0 +1,36 @@
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
/// <summary>
/// The parsed <c>OpcUa:Redundancy:Mode</c> (Gitea #717). Startup-static: changing it requires a restart.
/// </summary>
public enum OpcUaRedundancyMode
{
/// <summary>Not opted in (the default): byte-for-byte the pre-#717 server. Nothing in the redundancy metadata,
/// ServiceLevel bands, projection or alarm delivery changes.</summary>
Unset = 0,
/// <summary>Standalone: advertises <c>RedundancySupport=None</c> with no <c>ServerUriArray</c>; never projects.</summary>
None = 1,
/// <summary>A 2-node pair with a degraded standby (spec §4): non-transparent Warm metadata, Standby bands below
/// every Active level, ordinary-tag values projected <c>BadOutOfService</c> on the Standby, new alarm events
/// suppressed there, and writes / Part 9 methods refused <c>BadInvalidState</c> there.</summary>
Warm = 2,
}
/// <summary>
/// The runtime half of <c>OpcUa:Redundancy</c> (Gitea #717): what the redundancy actors need. The Host builds it
/// from the validated <c>OpcUaApplicationHostOptions</c> and registers it as a singleton; a node that registers
/// none (a harness, an admin-only node) is <see cref="Unset"/>.
/// </summary>
/// <param name="Mode">The redundancy mode.</param>
/// <param name="PromoteWhilePrimaryUnreachable">Warm only: let the survivor go Active while the Primary is
/// unreachable but not yet downed (spec §4.1, owner answer Q2). Default on.</param>
public sealed record WarmRedundancySettings(OpcUaRedundancyMode Mode, bool PromoteWhilePrimaryUnreachable = true)
{
/// <summary>The not-opted-in default.</summary>
public static WarmRedundancySettings Unset { get; } = new(OpcUaRedundancyMode.Unset);
/// <summary>True when this node runs the Warm role contract.</summary>
public bool IsWarm => Mode == OpcUaRedundancyMode.Warm;
}
@@ -37,6 +37,11 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
private readonly Action<object>? _broadcastOverride;
private readonly TimeSpan _heartbeatInterval;
private bool _dirty;
/// <summary>Gitea #717 spec §4.2: <c>Environment.TickCount64</c> at <see cref="PreStart"/>, ordering a singleton
/// restarted in the same process.</summary>
private long _incarnationTick;
/// <summary>Gitea #717 spec §4.2: incremented on every publish, heartbeats included.</summary>
private long _sequence;
/// <summary>Gets the timer scheduler for this actor.</summary>
public ITimerScheduler Timers { get; set; } = null!;
@@ -78,6 +83,7 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
/// <inheritdoc />
protected override void PreStart()
{
_incarnationTick = NextIncarnationTick();
_cluster.Subscribe(
Self,
ClusterEvent.InitialStateAsEvents,
@@ -107,12 +113,63 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
_dirty = false;
var snapshot = BuildSnapshot();
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId());
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId(), NextRevision());
if (_broadcastOverride is not null) _broadcastOverride(msg);
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(Topic, msg));
_log.Debug("Published RedundancyStateChanged with {Count} nodes", snapshot.Count);
}
/// <summary>
/// The ordered revision of the next publish (Gitea #717 spec §4.2): the up-number of the member hosting this
/// singleton, this incarnation's start tick, and a per-publish sequence. Subscribers drop anything not newer,
/// so a late heartbeat from an old incarnation can no longer overwrite the new incarnation's snapshot.
/// </summary>
private SnapshotRevision NextRevision()
{
long upNumber;
try
{
upNumber = ReadUpNumber(_cluster.SelfMember)
?? throw new InvalidOperationException("Akka.Cluster Member.UpNumber is not readable on this Akka version");
}
catch (Exception ex)
{
// Not expected on a running singleton; an unordered 0 can never outrank a real publisher.
_log.Debug(ex, "RedundancyState: SelfMember up-number unavailable; publishing revision up-number 0");
upNumber = 0;
}
return new SnapshotRevision(upNumber, _incarnationTick, ++_sequence);
}
private static long _lastIncarnationTick;
/// <summary><c>Environment.TickCount64</c>, made strictly increasing within the process so a singleton restarted
/// inside one tick still outranks its predecessor (whose sequence may be ahead of the new one's).</summary>
/// <returns>The incarnation tick.</returns>
public static long NextIncarnationTick()
{
while (true)
{
var last = Interlocked.Read(ref _lastIncarnationTick);
var next = Math.Max(Environment.TickCount64, last + 1);
if (Interlocked.CompareExchange(ref _lastIncarnationTick, next, last) == last) return next;
}
}
private static readonly System.Reflection.PropertyInfo? UpNumberProperty = typeof(Member).GetProperty(
"UpNumber", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic);
/// <summary>
/// A member's up-number — the order in which members became Up, the basis of <c>Member.IsOlderThan</c> and of
/// singleton placement. Akka.NET 1.5 keeps the property <c>internal</c>, so it is read by reflection; the
/// <c>SnapshotRevisionTests</c> pin that it resolves, so an Akka upgrade that renames it fails loudly.
/// </summary>
/// <param name="member">The member.</param>
/// <returns>The up-number, or null if the property is unavailable.</returns>
public static int? ReadUpNumber(Member member) =>
UpNumberProperty?.GetValue(member) is int value ? value : null;
/// <summary>The cluster role that marks a node as carrying the driver runtime.</summary>
public const string DriverRole = RoleParser.Driver;
@@ -60,5 +60,10 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase<
|| lockout.MaxFailedAttemptsPerClient >= lockout.MaxFailedAttempts,
"OpcUa:CredentialLockout:MaxFailedAttemptsPerClient must be 0 (= 2 x MaxFailedAttempts) or at least "
+ "OpcUa:CredentialLockout:MaxFailedAttempts");
// Gitea #717 spec §3: Hot/unknown modes fail start; Warm needs distinct peers that do not include this server's
// own ApplicationUri; None needs no peers. The effective URI is re-checked after an ApplicationConfigPath load.
foreach (var failure in (o.Redundancy ?? new OpcUaRedundancyOptions()).Check(o.ApplicationUri, o.PeerApplicationUris))
builder.RequireThat(false, failure);
}
}
@@ -0,0 +1,27 @@
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// The one Gitea #717 startup rule the options validator cannot see: <c>OpcUa:Redundancy:Mode=Warm</c> needs the
/// <c>driver</c> role (the validator is registered only on driver nodes, so on any other node it would never run).
/// </summary>
public static class OpcUaRedundancyStartupGuard
{
/// <summary>Fails host start when a node without the driver role is configured Warm.</summary>
/// <param name="configuredMode">The raw <c>OpcUa:Redundancy:Mode</c> value.</param>
/// <param name="hasDriver">Whether this node carries the driver role.</param>
/// <exception cref="InvalidOperationException">Warm without the driver role.</exception>
public static void RequireDriverRoleForWarm(string? configuredMode, bool hasDriver)
{
if (hasDriver) return;
var options = new OpcUaRedundancyOptions { Mode = configuredMode ?? string.Empty };
if (options.TryParseMode(out var mode) && mode == OpcUaRedundancyMode.Warm)
{
throw new InvalidOperationException(
"OpcUa:Redundancy:Mode is Warm on a node without the driver role; Warm redundancy needs the driver role "
+ "(the OPC UA endpoint is served by driver nodes only). Remove the setting or add the driver role.");
}
}
}
@@ -36,6 +36,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
private readonly OpcUaApplicationHostOptions _options;
private readonly DeferredAddressSpaceSink _deferredSink;
private readonly DeferredServiceLevelPublisher _deferredServiceLevel;
private readonly DeferredStandbyProjection? _deferredStandby;
private readonly IOpcUaUserAuthenticator _userAuthenticator;
private readonly IHistorianDataSource _historianDataSource;
private readonly IDataPlaneAuthoritySource _dataPlaneAuthority;
@@ -127,6 +128,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
/// <param name="writeBounds">The node's configured write bounds (<c>DriverWrites:Timeout</c>); the inbound
/// write gateway's Ask is derived from them (lmxopcua#777). Optional: null ⇒
/// <see cref="DriverWriteBounds.Default"/>.</param>
/// <param name="deferredStandby">The late-bound Warm standby projection (Gitea #717) <c>OpcUaPublishActor</c> drives;
/// bound here once the SDK has started. Optional: null (and a non-Warm server) binds nothing.</param>
public OtOpcUaServerHostedService(
IOptions<OpcUaApplicationHostOptions> options,
DeferredAddressSpaceSink deferredSink,
@@ -147,8 +150,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
OpcUaEndpointState? endpointState = null,
EffectiveRoles? roles = null,
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null)
DriverWriteBounds? writeBounds = null,
DeferredStandbyProjection? deferredStandby = null)
{
_deferredStandby = deferredStandby;
ArgumentNullException.ThrowIfNull(aclOptions);
ArgumentNullException.ThrowIfNull(time);
@@ -402,6 +407,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
serverInternal,
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
}
// Gitea #717: the Warm standby projection. Bound right after the ServiceLevel so a transition the actor
// already decided replays onto the node manager; null (every non-Warm server) leaves the adapter unbound.
if (_server.CreateStandbyProjection() is { } standbyProjection) _deferredStandby?.SetInner(standbyProjection);
}
catch (Exception ex)
{
@@ -568,6 +577,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
// half-disposed NodeManager.
_deferredSink.SetSink(null);
_deferredServiceLevel.SetInner(null);
_deferredStandby?.SetInner(null);
// Restore the Null write gateway so a late client write doesn't Ask a stopping DriverHostActor.
_server?.SetNodeWriteGateway(null);
// Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor.
@@ -206,6 +206,10 @@ ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.OpcUaUserCertificatePaths.Configure(
// lmxopcua#765: refresh/staleness of the cached API-key → Galaxy ack identity set (driver nodes consume it).
builder.Services.AddGalaxyAckIdentityCache(builder.Configuration);
// Gitea #717 spec §3: Warm redundancy is a property of the OPC UA endpoint, which only a driver node serves. A Warm
// mode on a node without the driver role is a configuration error, not something to ignore silently.
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(builder.Configuration["OpcUa:Redundancy:Mode"], hasDriver);
if (hasDriver)
{
builder.Services.AddOtOpcUaRuntime();
@@ -501,6 +505,11 @@ if (hasDriver)
builder.Services.AddValidatedOptions<OpcUaApplicationHostOptions, OpcUaApplicationHostOptionsValidator>(
builder.Configuration, "OpcUa");
// Gitea #717: the runtime half of OpcUa:Redundancy for the redundancy actors, and the late-bound standby
// projection the OPC UA hosted service binds once the SDK has started (same pattern as the ServiceLevel publisher).
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<OpcUaApplicationHostOptions>>().Value.Redundancy.ToSettings());
builder.Services.AddSingleton<DeferredStandbyProjection>();
// Lazy ActorSystem accessor so OtOpcUaServerHostedService can resolve the DistributedPubSub
// mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
// DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
@@ -6,8 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// Preserves the application's established equipment-write conversions across SDK 2.0's
/// new pre-handler type validation. The owning callback remains the authorization and shape gate.
/// </summary>
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock) : BaseDataVariableState(parent)
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock, WarmStandbyState? standby = null)
: BaseDataVariableState(parent)
{
private readonly WarmStandbyState _standby = standby ?? WarmStandbyState.Off;
// Every field below is accessed under ownerLock. Tokens identify client attempts; the mutation
// revision detects a driver publication that re-enters the callback before its optimistic commit.
private long _nextWriteToken;
@@ -82,6 +85,16 @@ internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock
QualifiedName dataEncoding, ref Variant value, ref DateTimeUtc sourceTimestamp)
{
// Subscription samplers read the node directly, bypassing the manager's service lock.
lock (ownerLock) return base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
ServiceResult result;
lock (ownerLock) result = base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
// Gitea #717 spec §4.4 (T0b): on a Warm Standby every value path — Read (IndexRange and DataEncoding included),
// the initial sample, the change fan-out, ResendData, AggregateFilter input — answers BadOutOfService/null. The
// stored Value is untouched and keeps updating from acquisition. The base read runs first, so an IndexRange or
// encoding error is still reported as such. Off (Unset/None/Active): returned unchanged.
if (ServiceResult.IsBad(result) || !_standby.DataProjected) return result;
value = Variant.Null;
sourceTimestamp = DateTimeUtc.MinValue;
return StatusCodes.BadOutOfService;
}
}
@@ -4,6 +4,7 @@ using Opc.Ua;
using Opc.Ua.Configuration;
using Opc.Ua.Identity;
using Opc.Ua.Server;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
@@ -174,6 +175,83 @@ public sealed class OpcUaApplicationHostOptions
/// Replaces the SDK's own lockout, which also refused Anonymous; see <see cref="FailedCredentialLockout"/>.
/// </summary>
public OpcUaCredentialLockoutOptions CredentialLockout { get; set; } = new();
/// <summary>
/// Warm non-transparent redundancy (<c>OpcUa:Redundancy:*</c>, Gitea #717). The default <c>Mode=Unset</c> is
/// byte-for-byte today's server. Startup-static.
/// </summary>
public OpcUaRedundancyOptions Redundancy { get; set; } = new();
}
/// <summary>The <c>OpcUa:Redundancy</c> section (Gitea #717, spec §6).</summary>
public sealed class OpcUaRedundancyOptions
{
/// <summary>
/// <c>Unset</c> (default) / <c>None</c> / <c>Warm</c>, case-insensitive. <c>Hot</c> is not advertisable
/// (spec §3) and, like any other value, fails host start. Bound as a string so a bad value is a readable
/// validation failure rather than a binder exception.
/// </summary>
public string Mode { get; set; } = nameof(OpcUaRedundancyMode.Unset);
/// <summary>
/// Warm only: let the survivor go Active while the Primary is unreachable but not yet downed (spec §4.1, owner
/// answer Q2). Default <c>true</c>.
/// </summary>
public bool PromoteWhilePrimaryUnreachable { get; set; } = true;
/// <summary>Parses <see cref="Mode"/>.</summary>
/// <param name="mode">The parsed mode.</param>
/// <returns><c>false</c> for <c>Hot</c>, numerics or any unknown value.</returns>
public bool TryParseMode(out OpcUaRedundancyMode mode)
{
var text = (Mode ?? string.Empty).Trim();
if (text.Length == 0) text = nameof(OpcUaRedundancyMode.Unset);
if (!text.All(char.IsLetter)) { mode = default; return false; }
return Enum.TryParse(text, ignoreCase: true, out mode) && Enum.IsDefined(mode);
}
/// <summary>The parsed mode; throws on an invalid one (the validator rejects it first).</summary>
public OpcUaRedundancyMode ParsedMode => TryParseMode(out var m)
? m
: throw new InvalidOperationException(
$"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable).");
/// <summary>The runtime settings the redundancy actors read.</summary>
/// <returns>The settings.</returns>
public WarmRedundancySettings ToSettings() => new(ParsedMode, PromoteWhilePrimaryUnreachable);
/// <summary>
/// The rules of spec §3 that need only configuration: <c>Warm</c> needs at least one peer, all URIs distinct
/// and the self URI not listed as a peer; <c>None</c> needs no peers. Returns the failure messages (empty when
/// valid). The effective self URI is checked again after an <c>ApplicationConfigPath</c> load.
/// </summary>
/// <param name="selfUri">The self URI to check against.</param>
/// <param name="peers">The configured peer URIs.</param>
/// <returns>The failures.</returns>
public IReadOnlyList<string> Check(string? selfUri, IEnumerable<string>? peers)
{
var failures = new List<string>();
if (!TryParseMode(out var mode))
{
failures.Add($"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable: continuous acquisition alone does not meet Part 4 Hot)");
return failures;
}
var list = (peers ?? Array.Empty<string>()).Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToList();
if (mode == OpcUaRedundancyMode.None && list.Count > 0)
failures.Add("OpcUa:PeerApplicationUris must be empty when OpcUa:Redundancy:Mode is None (a standalone server has no peers)");
if (mode == OpcUaRedundancyMode.Warm)
{
if (list.Count == 0)
failures.Add("OpcUa:Redundancy:Mode Warm requires at least one OpcUa:PeerApplicationUris entry");
if (list.Distinct(StringComparer.Ordinal).Count() != list.Count)
failures.Add("OpcUa:PeerApplicationUris must not contain duplicates under OpcUa:Redundancy:Mode Warm");
if (!string.IsNullOrWhiteSpace(selfUri) && list.Contains(selfUri.Trim(), StringComparer.Ordinal))
failures.Add($"OpcUa:PeerApplicationUris must not list this server's own ApplicationUri '{selfUri}' under OpcUa:Redundancy:Mode Warm (each pair node needs a unique ApplicationUri)");
}
return failures;
}
}
/// <summary>
@@ -289,7 +367,8 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
try
{
_ = await BuildConfigurationAsync(cancellationToken);
var configuration = await BuildConfigurationAsync(cancellationToken);
ConfigureRedundancy(sdkServer, configuration.ApplicationUri ?? _options.ApplicationUri);
await EnsureApplicationCertificateAsync(cancellationToken).ConfigureAwait(false);
AttachUserAuthenticator(sdkServer);
await _application.StartAsync(server, cancellationToken).ConfigureAwait(false);
@@ -305,6 +384,31 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
_options.PublicHostname, _options.OpcUaPort);
}
/// <summary>
/// Gitea #717 (spec §3): validate the redundancy mode against the EFFECTIVE self URI — the loaded
/// <c>ApplicationConfigPath</c> XML wins over <c>OpcUa:ApplicationUri</c> (c7787) — log the boot banner and hand
/// the startup-static metadata to the SDK server, which installs it in <c>OnNodeManagerStartedAsync</c>.
/// <c>Unset</c> hands nothing over, so the SDK's graph is untouched.
/// </summary>
/// <param name="server">The SDK server about to start.</param>
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
/// <exception cref="InvalidOperationException">The redundancy configuration is invalid for the effective URI.</exception>
private void ConfigureRedundancy(OtOpcUaSdkServer server, string effectiveSelfUri)
{
var redundancy = _options.Redundancy ?? new OpcUaRedundancyOptions();
var failures = redundancy.Check(effectiveSelfUri, _options.PeerApplicationUris);
if (failures.Count > 0)
throw new InvalidOperationException("OPC UA redundancy configuration is invalid: " + string.Join("; ", failures));
var mode = redundancy.ParsedMode;
var peers = (_options.PeerApplicationUris ?? new List<string>())
.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToArray();
_logger.LogInformation(
"OPC UA redundancy: Mode={Mode}, effective ApplicationUri={SelfUri}, peers=[{Peers}], PromoteWhilePrimaryUnreachable={Promote}",
mode, effectiveSelfUri, string.Join(", ", peers), redundancy.PromoteWhilePrimaryUnreachable);
server.ConfigureRedundancy(mode, effectiveSelfUri, peers);
}
private void AttachUserAuthenticator(OtOpcUaSdkServer server)
{
foreach (var type in new[] { UserTokenType.Anonymous, UserTokenType.UserName })
@@ -161,6 +161,34 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
private FolderState? _root;
private readonly ILogger<OtOpcUaNodeManager> _logger;
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
/// <param name="server">The OPC UA server instance.</param>
/// <param name="configuration">The application configuration.</param>
/// <param name="logger">Logger for the manager's swallowed-fault and skipped-wiring diagnostics;
/// <c>null</c> (tests) falls back to <see cref="NullLogger{T}.Instance"/>.</param>
/// <param name="standby">The Warm standby flags (Gitea #717); null (every non-Warm server) is
/// <see cref="WarmStandbyState.Off"/>, which leaves every data path exactly as it was.</param>
internal OtOpcUaNodeManager(
IServerInternal server, ApplicationConfiguration configuration, ILogger<OtOpcUaNodeManager>? logger,
WarmStandbyState? standby)
: this(server, configuration, logger)
{
_standby = standby ?? WarmStandbyState.Off;
if (_standby.WarmMode)
{
// Spec §4.6 item 1: ordinary-tag Value items are built by OrdinaryTagMonitoredItem, which marks each
// notification. Warm only — Unset/None keep the SDK's own manager.
m_monitoredItemManager.Dispose();
m_monitoredItemManager = new OrdinaryTagItemManager(this.ToAsyncNodeManager(), server);
}
}
/// <summary>The Warm standby flags (Gitea #717). <see cref="WarmStandbyState.Off"/> unless the server is Warm.</summary>
private readonly WarmStandbyState _standby = WarmStandbyState.Off;
/// <summary>Test observation: the Warm standby flags.</summary>
internal WarmStandbyState Standby => _standby;
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
/// <param name="server">The OPC UA server instance.</param>
/// <param name="configuration">The application configuration.</param>
@@ -1434,6 +1462,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
// the InstanceStateSnapshot captures the just-projected values.
alarm.ClearChangeMasks(SystemContext, includeChildren: true);
// Gitea #717 spec §4.5: on a Warm Standby only the STAMP half above runs — the condition node is current and a
// later refresh carries a valid, ackable EventId — and the DELIVER half below does not. Suppressing here, before
// the snapshot and every SDK field projection, covers the Server-object, folder and direct paths, ACL on or
// off, by construction; overflow, filtered retain and the #838 deferral queue never see the event. The
// decision is made at report time: an event reported while Active is delivered even if that ends after
// demotion.
if (_standby.AlarmsSuppressed)
{
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbySuppressedAlarmEvents.Add(1);
return;
}
try
{
// InstanceStateSnapshot is the IFilterTarget — a frozen copy of the condition's fields at fire
@@ -2991,6 +3031,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
var gate = EvaluateAlarmCommandGate(identity, condition, operation);
if (gate is not null) return gate;
// Gitea #717 owner answer Q4: a Warm Standby refuses every Part 9 condition method before it changes any state
// (nothing routed, nothing applied, no event). Unset/None keep today's acceptance.
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal($"call {operation}");
var cmd = new AlarmCommand(
AlarmId: condition.NodeId.IdentifierAsString,
Operation: operation,
@@ -3032,6 +3076,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
var gate = EvaluateAlarmCommandGate(identity, condition, "Acknowledge");
if (gate is not null) return gate;
// Gitea #717 owner answer Q4: refused on a Warm Standby, ahead of (and whatever) the Primary gate below says.
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("acknowledge the alarm");
// PRIMARY GATE, after authorization (a caller who may not ack learns nothing about the
// node's role). Only the Primary routes a native ack upstream (DriverHostActor applies the same verdict
// to the route), so on any other node the ack is refused outright rather than applied locally and then
@@ -3215,6 +3262,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
?? EvaluateEquipmentWriteGate(identity, gatewayWired);
if (gate is not null) return gate;
// Gitea #717 owner answer Q1: a Warm Standby refuses the write synchronously, AFTER authorization (a caller who
// may not write learns nothing about the node's role) and before anything reaches the driver. Nothing is
// applied, so nothing is reverted. Unset/None never set the flag and keep the #5 optimistic path.
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("write the value");
// Item A (synchronous structural fail-fast): reject a structurally-invalid write INLINE — return Bad
// synchronously so the SDK never applies it (no optimistic-Good-then-revert round-trip + no needless
// device dispatch). Runs AFTER the authz gate (so we never leak structure detail to an unauthorised
@@ -4120,7 +4172,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
// The SDK exposes the flags separately (no CurrentReadWrite composite): ReadWrite is
// CurrentRead | CurrentWrite, with the HistoryRead bit OR-ed in when historized.
var access = ComposeAccessLevel(writable, historized, HistoryWritable(_historianDataUpdater));
var variable = new EquipmentVariableState(parent, Lock)
var variable = new EquipmentVariableState(parent, Lock, _standby)
{
NodeId = new NodeId(variableNodeId, ns),
BrowseName = new QualifiedName(variableNodeId, ns),
@@ -5725,6 +5777,88 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
lock (Lock) return MonitoredNodes.ContainsKey(nodeId);
}
/// <summary>
/// Gitea #717 spec §4.7 promotion step 2 / demotion step 5: queues one value to every live, non-stood-down
/// ordinary-tag <c>Value</c> item, ignoring its filters (the queue pattern of <see cref="RevalidateMonitoredItems"/>
/// — never detach or <c>Disabled</c>, which drops the queue). Promotion queues the current stored value read
/// through the item's own IndexRange/DataEncoding; demotion queues <c>BadOutOfService</c>/null.
/// </summary>
/// <param name="projected">Queue the projected value (demotion) instead of the stored one (promotion).</param>
/// <returns>The number of items queued to.</returns>
internal int QueueCurrentToLiveItems(bool projected)
{
var queued = 0;
foreach (var (item, tracked) in _liveItems)
{
if (tracked.Suppressed || tracked.Handle.Node is not EquipmentVariableState variable) continue;
if (item is not IDataChangeMonitoredItem2 { AttributeId: Attributes.Value } data) continue;
var now = UtcNow;
DataValue value;
if (projected)
{
value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, now);
}
else
{
var stored = new DataValue();
ServiceResult read;
lock (Lock) read = variable.ReadAttribute(SystemContext, Attributes.Value, data.IndexRange, data.DataEncoding, ref stored);
value = ServiceResult.IsBad(read)
? new DataValue(Variant.Null, read.StatusCode, DateTimeUtc.MinValue, now)
: new DataValue(stored.WrappedValue, stored.StatusCode, stored.SourceTimestamp, now);
}
try
{
item.QueueValue(value, StatusCode.IsBad(value.StatusCode) ? new ServiceResult(value.StatusCode) : null, ignoreFilters: true);
queued++;
}
catch (Exception ex)
{
// A disabled item refuses the value; it re-samples the (projected or not) node when re-enabled.
_logger.LogDebug(ex, "OtOpcUaNodeManager: could not queue a Warm transition value to a monitored item");
}
}
return queued;
}
/// <summary>
/// Gitea #717 spec §4.5 promotion refresh: a ConditionRefresh2 on every live event item — Server-object items
/// and the direct/folder items of this manager's notifiers — through the SDK's own subscription path, so each
/// client sees RefreshStart, the retained conditions (through this manager's ACL decision and the #854/#855
/// cut), then RefreshEnd. Must not be called under <c>Lock</c>.
/// </summary>
/// <returns>The number of event items refreshed.</returns>
internal int RefreshAllEventItems()
{
var items = new List<IEventMonitoredItem>(_allEventsItems.Keys);
foreach (var (item, tracked) in _eventItems)
if (!tracked.Suppressed) items.Add(item);
if (items.Count == 0) return 0;
var subscriptions = new Dictionary<uint, ISubscription>();
foreach (var subscription in Server.SubscriptionManager.GetSubscriptions()) subscriptions[subscription.Id] = subscription;
var refreshed = 0;
foreach (var item in items)
{
if (!subscriptions.TryGetValue(item.SubscriptionId, out var subscription)) continue;
try
{
subscription.ConditionRefresh2Async(item.Id).AsTask().GetAwaiter().GetResult();
refreshed++;
}
catch (Exception ex)
{
// A subscription deleted mid-walk, or an item whose mode refuses a refresh: that client re-refreshes itself.
_logger.LogDebug(ex, "OtOpcUaNodeManager: Warm promotion refresh of event item {ItemId} failed", item.Id);
}
}
return refreshed;
}
/// <summary>Test observability: how many data-change items the revalidation map tracks.</summary>
internal int LiveItemCount => _liveItems.Count;
@@ -315,6 +315,160 @@ public sealed class OtOpcUaSdkServer : StandardServer
/// </summary>
public const byte StartupServiceLevel = 0;
private OpcUaRedundancyMode _redundancyMode = OpcUaRedundancyMode.Unset;
private string _selfUri = string.Empty;
private string[] _peerUris = [];
private WarmStandbyState _standby = WarmStandbyState.Off;
/// <summary>The redundancy mode configured before start (Gitea #717).</summary>
public OpcUaRedundancyMode RedundancyMode => _redundancyMode;
/// <summary>The Warm standby flags shared with the node manager.</summary>
internal WarmStandbyState Standby => _standby;
/// <summary>The initial <c>Server.ServiceLevel</c> under <c>Warm</c> (spec §4.3: the NoData band, replacing the
/// SDK's 255 and this server's #885 value 0 until the first computed level).</summary>
public const byte WarmBootServiceLevel = 1;
/// <summary>
/// Sets the startup-static redundancy metadata (Gitea #717 spec §3) before <c>StartAsync</c>.
/// <see cref="OpcUaRedundancyMode.Unset"/> (the default when this is never called) leaves the SDK graph and
/// every data path exactly as they are today.
/// </summary>
/// <param name="mode">The validated mode.</param>
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
/// <param name="peerUris">The peer URIs, in configured order.</param>
public void ConfigureRedundancy(OpcUaRedundancyMode mode, string effectiveSelfUri, IReadOnlyList<string> peerUris)
{
if (_otOpcUaNodeManager is not null)
throw new InvalidOperationException("Redundancy must be configured before the server starts.");
_redundancyMode = mode;
_selfUri = effectiveSelfUri;
_peerUris = peerUris.ToArray();
_standby = new WarmStandbyState(mode == OpcUaRedundancyMode.Warm);
}
/// <summary>
/// The data-plane transition steps a Warm <c>OpcUaPublishActor</c> drives (spec §4.7), or null when the server
/// is not Warm or has not started.
/// </summary>
/// <returns>The projection, or null.</returns>
public IStandbyProjection? CreateStandbyProjection() =>
_standby.WarmMode && _otOpcUaNodeManager is { } nm ? new SdkStandbyProjection(nm, _standby) : null;
/// <summary>Test seam: ids of the live subscriptions, for the decision record's dead-subscription sweep.</summary>
private IReadOnlySet<uint> LiveSubscriptionIds()
{
var ids = new HashSet<uint>();
try
{
if (CurrentInstance?.SubscriptionManager is { } manager)
foreach (var subscription in manager.GetSubscriptions()) ids.Add(subscription.Id);
}
catch (Exception)
{
// Shutdown race: an empty set only drops records, which re-decide (fail-safe) at their next delivery.
}
return ids;
}
private long _publishesSinceSweep;
/// <inheritdoc />
public override async ValueTask<PublishResponse> PublishAsync(SecureChannelContext secureChannelContext,
RequestHeader? requestHeader, ArrayOf<SubscriptionAcknowledgement> subscriptionAcknowledgements,
RequestLifetime requestLifetime)
{
var response = await base.PublishAsync(secureChannelContext, requestHeader, subscriptionAcknowledgements, requestLifetime)
.ConfigureAwait(false);
if (_standby.Delivery is not { } record || response?.NotificationMessage is not { } message) return response!;
if (message.NotificationData.Count > 0)
{
response.NotificationMessage = Decide(record, response.SubscriptionId, message);
var available = new List<uint>(response.AvailableSequenceNumbers.Count);
for (var i = 0; i < response.AvailableSequenceNumbers.Count; i++) available.Add(response.AvailableSequenceNumbers[i]);
record.Prune(response.SubscriptionId, available, message.SequenceNumber);
}
if (Interlocked.Increment(ref _publishesSinceSweep) % 1024 == 0) record.SweepDead(LiveSubscriptionIds());
return response;
}
/// <inheritdoc />
public override async ValueTask<RepublishResponse> RepublishAsync(SecureChannelContext secureChannelContext,
RequestHeader? requestHeader, uint subscriptionId, uint retransmitSequenceNumber, RequestLifetime requestLifetime)
{
var response = await base.RepublishAsync(secureChannelContext, requestHeader, subscriptionId, retransmitSequenceNumber,
requestLifetime).ConfigureAwait(false);
if (_standby.Delivery is { } record && response?.NotificationMessage is { NotificationData.Count: > 0 } message)
response.NotificationMessage = Decide(record, subscriptionId, message);
return response!;
}
/// <summary>The recorded (or, first time, current) decision for one delivery, applied to the SDK's fresh clone.</summary>
private NotificationMessage Decide(FirstDeliveryRecord record, uint subscriptionId, NotificationMessage message)
{
if (!record.Decide(subscriptionId, message.SequenceNumber)) return message;
var projected = StandbyNotificationRewriter.Project(message, out var rewritten);
if (rewritten > 0)
{
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbyProjectedNotifications.Add(rewritten);
}
return projected;
}
/// <summary>
/// Installs the standard redundancy metadata (Gitea #717 spec §3, T0a): under <c>Warm</c>
/// <c>Server.ServerRedundancy</c> becomes a <c>NonTransparentRedundancyType</c> whose read-only
/// <c>ServerUriArray</c> (<c>i=11314</c>) is <c>[self, peers…]</c>, <c>RedundancySupport</c> is <c>Warm</c> and
/// ServiceLevel boots at <see cref="WarmBootServiceLevel"/>; under <c>None</c> <c>RedundancySupport</c> is
/// <c>None</c> and the SDK's orphan <c>i=11314</c> is deleted, so it is absent by NodeId as well as by browse.
/// <c>Unset</c> touches nothing.
/// </summary>
private async ValueTask InstallRedundancyMetadataAsync(IServerInternal server, CancellationToken cancellationToken)
{
if (_redundancyMode == OpcUaRedundancyMode.Unset) return;
var serverObject = server.ServerObject
?? throw new InvalidOperationException("The Server object is unavailable during server startup.");
var context = server.DefaultSystemContext;
var redundancy = serverObject.ServerRedundancy
?? throw new InvalidOperationException("Server.ServerRedundancy is unavailable during server startup.");
if (_redundancyMode == OpcUaRedundancyMode.None)
{
redundancy.RedundancySupport!.Value = RedundancySupport.None;
redundancy.RedundancySupport.ClearChangeMasks(context, false);
// T0a finding: the SDK's predefined i=11314 survives as an orphan (unreferenced, but Good [] by NodeId).
if (server.DiagnosticsNodeManager is AsyncCustomNodeManager diagnostics)
await diagnostics.DeleteNodeAsync(context, VariableIds.Server_ServerRedundancy_ServerUriArray, cancellationToken)
.ConfigureAwait(false);
return;
}
var replacer = server.DiagnosticsNodeManager as IPredefinedNodeSubtypeReplacer
?? throw new InvalidOperationException("The diagnostics node manager cannot replace predefined node subtypes.");
var replaced = (NonTransparentRedundancyState)await replacer.ReplacePredefinedInstanceSubtypeAsync(context, redundancy,
context.CreateInstanceOfNonTransparentRedundancyType(),
new Dictionary<QualifiedName, NodeId>
{
[new QualifiedName(BrowseNames.ServerUriArray, 0)] = VariableIds.Server_ServerRedundancy_ServerUriArray,
},
node => serverObject.ServerRedundancy = (ServerRedundancyState)node, cancellationToken).ConfigureAwait(false);
replaced.RedundancySupport!.Value = RedundancySupport.Warm;
replaced.ServerUriArray!.Value = (ArrayOf<string>)new[] { _selfUri }.Concat(_peerUris).ToArray();
replaced.ServerUriArray.AccessLevel = replaced.ServerUriArray.UserAccessLevel = AccessLevels.CurrentRead;
replaced.ClearChangeMasks(context, true);
if (serverObject.ServiceLevel is { } serviceLevel)
{
serviceLevel.Value = WarmBootServiceLevel;
serviceLevel.Timestamp = DateTime.UtcNow;
serviceLevel.ClearChangeMasks(context, includeChildren: false);
}
}
/// <inheritdoc />
protected override async ValueTask OnNodeManagerStartedAsync(
IServerInternal server, CancellationToken cancellationToken = default)
@@ -329,6 +483,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
serviceLevel.Timestamp = DateTime.UtcNow;
serviceLevel.ClearChangeMasks(server.DefaultSystemContext, includeChildren: false);
}
await InstallRedundancyMetadataAsync(server, cancellationToken).ConfigureAwait(false);
var auditType = server.DiagnosticsNodeManager?
.FindPredefinedNode<BaseObjectTypeState>(ObjectTypeIds.AuditWriteUpdateEventType)
?? throw new InvalidOperationException("AuditWriteUpdateEventType is unavailable during server startup.");
@@ -360,7 +515,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
protected override MasterNodeManager CreateMasterNodeManager(
IServerInternal server, ApplicationConfiguration configuration)
{
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger);
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger, _standby);
return new MasterNodeManager(server, configuration, dynamicNamespaceUri: null, _otOpcUaNodeManager);
}
}
@@ -0,0 +1,388 @@
using Microsoft.Extensions.Logging;
using Opc.Ua;
using Opc.Ua.Server;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <summary>
/// The node-manager-wide Warm standby flags (Gitea #717, spec §4.4–§4.6). One instance per server. Under
/// <c>Unset</c>/<c>None</c> <see cref="WarmMode"/> is false and nothing ever sets a flag, so every reader sees
/// today's behaviour. Under <c>Warm</c> the node starts Standby (all flags set) until the redundancy layer
/// promotes it.
/// </summary>
internal sealed class WarmStandbyState
{
private volatile bool _dataProjected;
private volatile bool _alarmsSuppressed;
/// <summary>Initializes the state.</summary>
/// <param name="warmMode">Whether this server runs the Warm role contract.</param>
public WarmStandbyState(bool warmMode)
{
WarmMode = warmMode;
_dataProjected = warmMode;
_alarmsSuppressed = warmMode;
Delivery = warmMode ? new FirstDeliveryRecord(standby: true) : null;
}
/// <summary>The not-Warm state (no projection, no suppression, no decision record).</summary>
public static WarmStandbyState Off { get; } = new(false);
/// <summary>Whether this server runs the Warm role contract.</summary>
public bool WarmMode { get; }
/// <summary>Ordinary-tag reads/samples project <c>BadOutOfService</c>; writes and Part 9 methods are refused.</summary>
public bool DataProjected
{
get => _dataProjected;
set => _dataProjected = WarmMode && value;
}
/// <summary>New alarm condition events are not delivered (still stamped).</summary>
public bool AlarmsSuppressed
{
get => _alarmsSuppressed;
set => _alarmsSuppressed = WarmMode && value;
}
/// <summary>Whether a Warm Standby must refuse a write or a Part 9 method (owner answers Q1/Q4).</summary>
public bool RefusesMutations => WarmMode && _dataProjected;
/// <summary>The first-delivery decision record; null when not Warm.</summary>
public FirstDeliveryRecord? Delivery { get; }
/// <summary>The refusal returned to a write or a Part 9 method on a Warm Standby.</summary>
public static ServiceResult StandbyRefusal(string what) => new(StatusCodes.BadInvalidState,
new LocalizedText($"This server is the Warm redundancy standby; {what} on the active server (the one with the higher ServiceLevel)."));
}
/// <summary>
/// The ordinary-tag marker (spec §4.6 item 1): set on <see cref="MonitoredItemNotification.Message"/>, a helper
/// reference the binary encoder never writes and the SDK's per-delivery <c>CoreUtils.Clone</c> copies
/// (pinned by <c>WarmStandbySdkFieldContractTests</c>). Valid only for the notification whose
/// <see cref="ClientHandle"/> and <see cref="Value"/> it captured, so a stale marker left on a pooled payload
/// (2.0.1's <c>Initialize()</c> does not clear the helper) fails <see cref="Matches"/>.
/// </summary>
/// <remarks>Derives from <see cref="NotificationMessage"/> only because that is the helper's declared type.</remarks>
internal sealed class OrdinaryTagMarker : NotificationMessage
{
/// <summary>Initializes the marker.</summary>
/// <param name="clientHandle">The notification's client handle when it was built.</param>
/// <param name="value">The notification's final value when it was built.</param>
public OrdinaryTagMarker(uint clientHandle, DataValue value)
{
ClientHandle = clientHandle;
Value = value;
}
/// <summary>The client handle captured at build time.</summary>
public uint ClientHandle { get; }
/// <summary>The value captured at build time.</summary>
public DataValue Value { get; }
/// <summary>Whether <paramref name="n"/> is the notification this marker was made for.</summary>
/// <param name="n">A delivered notification.</param>
/// <returns>True for a match on client handle and value.</returns>
public bool Matches(MonitoredItemNotification n) => n.ClientHandle == ClientHandle && Equals(n.Value, Value);
/// <summary>Whether <paramref name="n"/> carries a valid ordinary-tag marker.</summary>
/// <param name="n">A delivered notification.</param>
/// <returns>True when it is an ordinary-tag notification.</returns>
public static bool IsOrdinaryTag(MonitoredItemNotification n) => n.Message is OrdinaryTagMarker m && m.Matches(n);
}
/// <summary>
/// The per-message first-delivery decision (spec §4.6 item 2, D4), keyed by
/// <c>(SubscriptionId, SequenceNumber)</c> — never by object: SDK 2.0.1 hands every Publish and Republish a fresh
/// clone of the cached message. The first delivery of a sequence number records Active or Standby; every later
/// delivery (Republish included) applies the recorded decision. One lock orders a decision against a role flip;
/// it is never held across an await or a projection.
/// </summary>
internal sealed class FirstDeliveryRecord
{
/// <summary>Recorded messages kept per subscription beyond what <c>AvailableSequenceNumbers</c> prunes.</summary>
internal const int MaxPerSubscription = 1024;
/// <summary>Subscriptions kept before the least-recently-used one is dropped (dead subscriptions).</summary>
internal const int MaxSubscriptions = 8192;
private readonly object _gate = new();
private readonly Dictionary<uint, SubscriptionRecord> _subscriptions = new();
private bool _standby;
private long _touch;
/// <summary>Initializes the record.</summary>
/// <param name="standby">The initial delivery state (a Warm node boots Standby).</param>
public FirstDeliveryRecord(bool standby) => _standby = standby;
/// <summary>Test seam: runs inside the lock after the decision is read and before it is recorded.</summary>
internal Action? DecisionBarrier { get; set; }
/// <summary>The current first-delivery state.</summary>
public bool Standby
{
get { lock (_gate) return _standby; }
}
/// <summary>Flips the first-delivery state (the D4 boundary). Waits for a decision in progress.</summary>
/// <param name="standby">Whether later first deliveries are Standby.</param>
public void SetStandby(bool standby)
{
lock (_gate) _standby = standby;
}
/// <summary>The decision for one delivery: the recorded one, or (first delivery) the current state, recorded.</summary>
/// <param name="subscriptionId">The subscription.</param>
/// <param name="sequenceNumber">The message's sequence number.</param>
/// <returns>True when the delivery must be projected (Standby).</returns>
public bool Decide(uint subscriptionId, uint sequenceNumber)
{
lock (_gate)
{
if (!_subscriptions.TryGetValue(subscriptionId, out var sub))
{
if (_subscriptions.Count >= MaxSubscriptions) EvictLeastRecentlyUsed();
_subscriptions[subscriptionId] = sub = new SubscriptionRecord();
}
sub.Touched = ++_touch;
if (sub.Decisions.TryGetValue(sequenceNumber, out var recorded)) return recorded;
var standby = _standby;
DecisionBarrier?.Invoke();
sub.Decisions[sequenceNumber] = standby;
sub.Order.Enqueue(sequenceNumber);
while (sub.Order.Count > MaxPerSubscription && sub.Order.TryDequeue(out var oldest)) sub.Decisions.Remove(oldest);
return standby;
}
}
/// <summary>Drops the decisions of messages the subscription no longer retains (they can never be republished).</summary>
/// <param name="subscriptionId">The subscription the Publish response belongs to.</param>
/// <param name="available">The response's <c>AvailableSequenceNumbers</c>.</param>
/// <param name="justDelivered">The sequence number just delivered (kept: it becomes available for Republish).</param>
public void Prune(uint subscriptionId, IReadOnlyCollection<uint> available, uint justDelivered)
{
lock (_gate)
{
if (!_subscriptions.TryGetValue(subscriptionId, out var sub)) return;
var keep = new HashSet<uint>(available) { justDelivered };
var kept = new Queue<uint>();
while (sub.Order.TryDequeue(out var seq))
{
if (keep.Contains(seq)) kept.Enqueue(seq);
else sub.Decisions.Remove(seq);
}
sub.Order.Clear();
foreach (var seq in kept) sub.Order.Enqueue(seq);
}
}
/// <summary>Drops every record of subscriptions not in <paramref name="live"/>.</summary>
/// <param name="live">The ids of live subscriptions.</param>
public void SweepDead(IReadOnlySet<uint> live)
{
lock (_gate)
{
foreach (var id in _subscriptions.Keys.Where(id => !live.Contains(id)).ToList()) _subscriptions.Remove(id);
}
}
/// <summary>Recorded decisions (test observation).</summary>
internal int Count
{
get { lock (_gate) return _subscriptions.Values.Sum(s => s.Decisions.Count); }
}
/// <summary>Recorded subscriptions (test observation).</summary>
internal int SubscriptionCount
{
get { lock (_gate) return _subscriptions.Count; }
}
private void EvictLeastRecentlyUsed()
{
var victim = _subscriptions.MinBy(kv => kv.Value.Touched).Key;
_subscriptions.Remove(victim);
}
private sealed class SubscriptionRecord
{
public readonly Dictionary<uint, bool> Decisions = new();
public readonly Queue<uint> Order = new();
public long Touched;
}
}
/// <summary>
/// Rewrites a Standby delivery (spec §4.6): every marked ordinary-tag notification whose status is not Bad becomes
/// <c>BadOutOfService</c>/null with its server timestamp kept. Builds new objects; the message handed in (the SDK's
/// fresh per-delivery clone) and, a fortiori, the SDK's cached copy are never mutated.
/// </summary>
internal static class StandbyNotificationRewriter
{
/// <summary>Projects <paramref name="message"/>, or returns it unchanged when nothing needs rewriting.</summary>
/// <param name="message">A delivered message.</param>
/// <param name="rewritten">The number of notifications rewritten.</param>
/// <returns>The message to deliver.</returns>
public static NotificationMessage Project(NotificationMessage message, out int rewritten)
{
rewritten = 0;
if (message is null || message.NotificationData.Count == 0) return message!;
var data = new List<ExtensionObject>(message.NotificationData.Count);
var changed = false;
for (var i = 0; i < message.NotificationData.Count; i++)
{
var eo = message.NotificationData[i];
if (eo.TryGetValue<DataChangeNotification>(out var dcn) && dcn is not null)
{
List<MonitoredItemNotification>? items = null;
for (var j = 0; j < dcn.MonitoredItems.Count; j++)
{
var n = dcn.MonitoredItems[j];
if (OrdinaryTagMarker.IsOrdinaryTag(n) && !StatusCode.IsBad(n.Value.StatusCode))
{
items ??= CopyPrefix(dcn, j);
items.Add(new MonitoredItemNotification
{
ClientHandle = n.ClientHandle,
Value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, n.Value.ServerTimestamp),
});
rewritten++;
}
else
{
items?.Add(n);
}
}
if (items is not null)
{
changed = true;
data.Add(new ExtensionObject(new DataChangeNotification { MonitoredItems = items, DiagnosticInfos = dcn.DiagnosticInfos }));
continue;
}
}
data.Add(eo);
}
if (!changed) return message;
return new NotificationMessage { SequenceNumber = message.SequenceNumber, PublishTime = message.PublishTime, NotificationData = data };
}
private static List<MonitoredItemNotification> CopyPrefix(DataChangeNotification dcn, int count)
{
var list = new List<MonitoredItemNotification>(dcn.MonitoredItems.Count);
for (var k = 0; k < count; k++) list.Add(dcn.MonitoredItems[k]);
return list;
}
}
/// <summary>
/// A sampled data-change item for an ordinary-tag <c>Value</c> (spec §4.6 item 1): after the SDK's own per-value
/// publish (so the timestamp filter has run) it marks each notification it produced with an
/// <see cref="OrdinaryTagMarker"/>. Everything else is the SDK's <see cref="MonitoredItem"/>.
/// </summary>
internal sealed class OrdinaryTagMonitoredItem : MonitoredItem
{
/// <summary>Initializes the item exactly as <see cref="MonitoredNodeMonitoredItemManager"/> does.</summary>
public OrdinaryTagMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager, NodeHandle handle, uint subscriptionId,
uint id, MonitoredItemCreateRequest request, DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn,
MonitoringFilter? filter, Opc.Ua.Range? range, double samplingInterval, uint queueSize, bool durable)
: base(server, nodeManager, handle, subscriptionId, id, request.ItemToMonitor, diagnosticsMasks, timestampsToReturn,
request.MonitoringMode, request.RequestedParameters.ClientHandle, filter, filter, range, samplingInterval, queueSize,
request.RequestedParameters.DiscardOldest, 0, durable)
{
}
/// <inheritdoc />
public override bool Publish(OperationContext context, Queue<MonitoredItemNotification> notifications,
Queue<DiagnosticInfo> diagnostics, uint maxNotificationsPerPublish, ILogger logger)
{
var local = new Queue<MonitoredItemNotification>();
var localDiagnostics = new Queue<DiagnosticInfo>();
var more = base.Publish(context, local, localDiagnostics, maxNotificationsPerPublish, logger);
while (local.Count > 0)
{
var n = local.Dequeue();
n.Message = new OrdinaryTagMarker(n.ClientHandle, n.Value);
notifications.Enqueue(n);
if (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
}
while (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
return more;
}
}
/// <summary>
/// Installs <see cref="OrdinaryTagMonitoredItem"/> for ordinary-tag (<see cref="EquipmentVariableState"/>)
/// <c>Value</c> items only. Re-implements <see cref="IMonitoredItemManager.CreateMonitoredItem"/> on the SDK's
/// public, non-sealed manager, so <c>CustomNodeManager2</c>'s <c>is MonitoredNodeMonitoredItemManager</c> checks and
/// its internal lifecycle interface keep working (T0c; the 2.0.1 <c>MonitoredItemFactory</c> hook exists only on
/// <c>AsyncCustomNodeManager</c>). Installed only under Warm; <c>Unset</c>/<c>None</c> keep the SDK's manager.
/// </summary>
internal sealed class OrdinaryTagItemManager(IAsyncNodeManager nodeManager, IServerInternal server)
: MonitoredNodeMonitoredItemManager(nodeManager, server), IMonitoredItemManager
{
/// <inheritdoc />
ISampledDataChangeMonitoredItem IMonitoredItemManager.CreateMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager,
ServerSystemContext context, NodeHandle handle, uint subscriptionId, double publishingInterval,
DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn, MonitoredItemCreateRequest itemToCreate,
Opc.Ua.Range euRange, MonitoringFilter filterToUse, double samplingInterval, uint revisedQueueSize, bool createDurable,
MonitoredItemIdFactory monitoredItemIdFactory, Func<ISystemContext, NodeHandle, NodeState, NodeState> addNodeToComponentCache)
{
if (handle.Node is not EquipmentVariableState || itemToCreate.ItemToMonitor.AttributeId != Attributes.Value)
{
return CreateMonitoredItem(server, nodeManager, context, handle, subscriptionId, publishingInterval, diagnosticsMasks,
timestampsToReturn, itemToCreate, euRange, filterToUse, samplingInterval, revisedQueueSize, createDurable,
monitoredItemIdFactory, addNodeToComponentCache);
}
// Mirrors MonitoredNodeMonitoredItemManager.CreateMonitoredItem in exact tagged 2.0.1, swapping only the item type.
var cachedNode = addNodeToComponentCache(context, handle, handle.Node);
if (!MonitoredNodes.TryGetValue(handle.Node.NodeId, out var monitoredNode))
{
MonitoredNodes[handle.Node.NodeId] = monitoredNode = new MonitoredNode2(nodeManager, server, cachedNode,
nodeManager.IsMultipleEventConsumerNode(cachedNode.NodeId));
}
handle.Node = monitoredNode.Node;
handle.MonitoredNode = monitoredNode;
uint id;
do
{
id = monitoredItemIdFactory.GetNextId();
}
while (!MonitoredItems.TryAdd(id, null!));
var item = new OrdinaryTagMonitoredItem(server, nodeManager, handle, subscriptionId, id, itemToCreate, diagnosticsMasks,
timestampsToReturn, filterToUse, euRange, samplingInterval, revisedQueueSize, createDurable);
monitoredNode.Add((IDataChangeMonitoredItem2)item);
MonitoredItems[id] = item;
return item;
}
}
/// <summary>The <see cref="ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection"/> a Warm server hands the redundancy layer.</summary>
internal sealed class SdkStandbyProjection(OtOpcUaNodeManager nodeManager, WarmStandbyState state)
: ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection
{
/// <inheritdoc />
public void SetDataProjection(bool projected) => state.DataProjected = projected;
/// <inheritdoc />
public void SetStandbyDelivery(bool standby) => state.Delivery?.SetStandby(standby);
/// <inheritdoc />
public void SetAlarmSuppression(bool suppressed) => state.AlarmsSuppressed = suppressed;
/// <inheritdoc />
public int QueueCurrentToLiveItems(bool projected) => nodeManager.QueueCurrentToLiveItems(projected);
/// <inheritdoc />
public int RefreshAllEventItems() => nodeManager.RefreshAllEventItems();
}
@@ -258,6 +258,52 @@ public static class DeploymentArtifact
}
}
/// <summary>
/// Gitea #717 spec §3: the <c>ServerCluster.RedundancyMode</c> of this node's cluster as the artifact carries it
/// (<c>None</c>/<c>Warm</c>/<c>Hot</c>), for the deploy-time cross-check against <c>OpcUa:Redundancy:Mode</c>.
/// A single-cluster artifact answers for its only cluster; a multi-cluster one for the cluster of this node's
/// <c>ClusterNode</c> row. Null when the artifact does not say (empty, malformed, no matching cluster).
/// </summary>
/// <param name="blob">The deployment artifact.</param>
/// <param name="nodeId">This node's <c>host:port</c> id.</param>
/// <returns>The mode name, or null.</returns>
public static string? ReadClusterRedundancyMode(ReadOnlySpan<byte> blob, string nodeId)
{
if (blob.IsEmpty) return null;
try
{
using var parsed = Open(blob.ToArray());
var root = parsed.Root;
if (!root.TryGetProperty("Clusters", out var clusters) || clusters.ValueKind != JsonValueKind.Array) return null;
var scope = parsed.ResolveClusterScope(nodeId);
JsonElement? match = null;
foreach (var cluster in clusters.EnumerateArray())
{
if (cluster.ValueKind != JsonValueKind.Object) continue;
if (scope.Mode == ClusterFilterMode.None) { match = cluster; break; }
if (scope.Mode == ClusterFilterMode.ScopeTo
&& string.Equals(cluster.TryGetProperty("ClusterId", out var cid) && cid.ValueKind == JsonValueKind.String ? cid.GetString() : null, scope.ClusterId, StringComparison.OrdinalIgnoreCase))
{
match = cluster;
break;
}
}
if (match is not { } m || !m.TryGetProperty("RedundancyMode", out var mode)) return null;
return mode.ValueKind switch
{
JsonValueKind.String => mode.GetString(),
JsonValueKind.Number when mode.TryGetInt32(out var n) && Enum.IsDefined(typeof(Configuration.Enums.RedundancyMode), n)
=> ((Configuration.Enums.RedundancyMode)n).ToString(),
_ => null,
};
}
catch (JsonException)
{
return null;
}
}
/// <summary>
/// Resolve how a node should scope a deployment artifact to its own ClusterId. Single-cluster
/// (or legacy) artifacts resolve to <see cref="ClusterFilterMode.None"/> so every existing
@@ -789,7 +789,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null) =>
TimeSpan? childShutdownTimeout = null,
bool enforceSnapshotOrdering = false) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -803,7 +804,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout));
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout,
enforceSnapshotOrdering));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -938,8 +940,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null)
TimeSpan? childShutdownTimeout = null,
bool enforceSnapshotOrdering = false)
{
_snapshotOrdering = enforceSnapshotOrdering ? new SnapshotRevisionGate() : null;
_childShutdownTimeout = childShutdownTimeout;
_scriptedAlarmRunner = scriptedAlarmRunner;
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
@@ -2552,8 +2556,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// the cached role unchanged; when this node has a real driver peer (count &gt; 1) that omission is the
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
/// <summary>Gitea #717: the snapshot order gate under Warm; null (no ordering, today's behaviour) otherwise.</summary>
private readonly SnapshotRevisionGate? _snapshotOrdering;
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
{
// Gitea #717 spec §4.2, under OpcUa:Redundancy:Mode=Warm only (Unset/None keep today's unconditional replace):
// a snapshot older than the last accepted one — a late heartbeat of an old singleton incarnation — is dropped,
// so it cannot overwrite the new incarnation's roles. An unrevisioned snapshot (pre-#717 publisher) still feeds
// the gates, as before.
if (_snapshotOrdering?.Offer(msg) == SnapshotRejection.OlderRevision)
{
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
return;
}
_redundancySnapshotReceived = true;
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
if (local is not null)
@@ -14,6 +14,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
@@ -326,7 +327,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null) =>
IOpcUaEndpointState? endpointState = null,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -341,7 +344,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
dbLess,
dataPlaneAuthority,
utcNow,
endpointState)).WithDispatcher(DispatcherId);
endpointState,
redundancy,
standbyProjection)).WithDispatcher(DispatcherId);
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
@@ -385,7 +390,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null) =>
IOpcUaEndpointState? endpointState = null,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -400,7 +407,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
dbLess,
dataPlaneAuthority,
utcNow,
endpointState));
endpointState,
redundancy,
standbyProjection));
/// <summary>Test-only Props exposing the deferred-write bound and the plan-step seams.</summary>
internal static Props PropsForTests(
@@ -461,10 +470,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null)
IOpcUaEndpointState? endpointState = null,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null)
: this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow,
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null,
endpointState)
endpointState, redundancy, standbyProjection)
{
}
@@ -488,8 +499,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Func<DateTime>? utcNow,
int deferredWriteCapacity,
TestHooks? hooks,
IOpcUaEndpointState? endpointState)
IOpcUaEndpointState? endpointState,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null)
{
_redundancy = redundancy ?? WarmRedundancySettings.Unset;
_standbyProjection = standbyProjection;
_deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity));
_hooks = hooks;
_sink = sink;
@@ -523,6 +538,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Receive<HealthTick>(_ => OnHealthTick());
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(HandlePeerProbe);
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
// Gitea #717 spec §4.1 / owner answer Q2: under Warm the verdict re-derives on membership and reachability, not
// only on snapshots. Nothing subscribes these under Unset/None.
Receive<ClusterEvent.IMemberEvent>(_ => RecomputeServiceLevel());
Receive<ClusterEvent.IReachabilityEvent>(_ => RecomputeServiceLevel());
Receive<ClusterEvent.CurrentClusterState>(_ => RecomputeServiceLevel());
Receive<WarmRetry>(_ => RecomputeServiceLevel());
}
/// <summary>
@@ -543,6 +564,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
base.PreRestart(reason, message);
}
/// <inheritdoc />
protected override void PostStop()
{
if (_redundancy.IsWarm)
{
try { _cluster.Unsubscribe(Self); }
catch (Exception) { /* the cluster is shutting down with us */ }
}
base.PostStop();
}
/// <inheritdoc />
protected override void PreStart()
{
@@ -551,6 +584,20 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
}
if (_redundancy.IsWarm)
{
OtOpcUaTelemetry.SetWarmRole(0);
try
{
_cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents,
typeof(ClusterEvent.IMemberEvent), typeof(ClusterEvent.IReachabilityEvent));
}
catch (Exception ex)
{
_log.Warning(ex, "OpcUaPublish: Warm redundancy could not subscribe to cluster membership; the verdict re-derives on snapshots only");
}
}
// DB-backed nodes refresh their cached probe status; DB-less nodes still need periodic
// snapshot/probe-verdict age checks even when no new messages arrive. Legacy nodes keep
// their role-only behavior without a timer.
@@ -672,7 +719,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
}
private void HandleRebuild(RebuildAddressSpace msg) => StartRebuild(msg, Sender);
private void HandleRebuild(RebuildAddressSpace msg)
{
if (msg.Artifact is { Length: > 0 } artifact) CheckRedundancyMode(artifact);
StartRebuild(msg, Sender);
}
/// <summary>
/// The entry half of a rebuild, on the actor thread. The raw-sink fallback (no applier) and the
@@ -1465,26 +1516,34 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
}
private void HandleServiceLevelChanged(ServiceLevelChanged msg)
private void HandleServiceLevelChanged(ServiceLevelChanged msg) => TryPublishServiceLevel(msg.ServiceLevel);
/// <summary>The dedup/publish/metric step behind <see cref="ServiceLevelChanged"/>; synchronous so a Warm
/// transition can order it against the projection steps (spec §4.7).</summary>
/// <param name="level">The level to publish.</param>
/// <returns>True when the level is published (or already was).</returns>
private bool TryPublishServiceLevel(byte level)
{
// Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
if (_publishedAtLeastOnce && level == _lastServiceLevel) return true;
try
{
_serviceLevel.Publish(msg.ServiceLevel);
_lastServiceLevel = msg.ServiceLevel;
_serviceLevel.Publish(level);
_lastServiceLevel = level;
_publishedAtLeastOnce = true;
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
_log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel);
new KeyValuePair<string, object?>("level", level));
_log.Debug("OpcUaPublish: ServiceLevel={Level}", level);
return true;
}
catch (Exception ex)
{
// A failed write may be partially applied; even a previous successful value must retry.
_publishedAtLeastOnce = false;
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", msg.ServiceLevel);
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", level);
return false;
}
}
@@ -1494,6 +1553,28 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// wired) or via the legacy role-only seam (back-compat / bootstrap).</summary>
private void HandleRedundancyStateChanged(RedundancyStateChanged msg)
{
if (_redundancy.IsWarm)
{
// Gitea #717 spec §4.2: under Warm a snapshot older than the last accepted one (a late heartbeat from an old
// singleton incarnation) is dropped outright. An unrevisioned one (a pre-#717 publisher mid-upgrade) still
// feeds the health inputs but can never make this node Active. Unset/None keep today's unconditional
// replace below.
switch (_revisionGate.Offer(msg))
{
case SnapshotRejection.OlderRevision:
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
_log.Debug("OpcUaPublish: dropped redundancy snapshot {Revision} (last accepted {Last})",
msg.Revision, _revisionGate.LastAccepted);
return;
case SnapshotRejection.Unrevisioned:
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "unrevisioned"));
break;
default:
_acceptedSnapshot = msg;
break;
}
}
_lastSnapshot = msg;
RecomputeServiceLevel();
}
@@ -1576,6 +1657,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
private void RecomputeServiceLevel()
{
if (_localNode is null || _lastSnapshot is null) return;
if (_redundancy.IsWarm)
{
RecomputeWarm();
return;
}
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
@@ -1635,6 +1721,197 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
}
// ---------------------------------------------------------------- Warm redundancy (Gitea #717)
private sealed class WarmRetry { public static readonly WarmRetry Instance = new(); private WarmRetry() { } }
private readonly WarmRedundancySettings _redundancy;
private readonly IStandbyProjection? _standbyProjection;
private readonly SnapshotRevisionGate _revisionGate = new();
/// <summary>The latest ACCEPTED revisioned snapshot — the only one the Warm verdict reads (spec §4.2).</summary>
private RedundancyStateChanged? _acceptedSnapshot;
/// <summary>The role the data plane is in. A Warm node manager boots Standby, so this starts Standby.</summary>
private WarmRole _warmRole = WarmRole.Standby;
/// <summary>A promotion whose Active ServiceLevel publish failed (spec §4.7): the data plane is Active and the
/// level is retried; the Standby level stays visible meanwhile, never the reverse.</summary>
private bool _promotionLevelPending;
private DateTime? _disagreeSinceUtc;
private bool _disagreementWarned;
/// <summary>How long membership and snapshot may disagree before a Warning (two heartbeats, spec §4.1).</summary>
internal static readonly TimeSpan WarmDisagreementWarning = TimeSpan.FromSeconds(20);
/// <summary>The current Warm role (test observation).</summary>
internal WarmRole CurrentWarmRole => _warmRole;
/// <summary>A promoted node whose Active ServiceLevel publish is being retried (test observation).</summary>
internal bool PromotionLevelPending => _promotionLevelPending;
/// <summary>
/// The Warm ServiceLevel + role path (spec §4.1, §4.3, §4.7): the health facts the non-Warm paths use, the
/// Warm verdict, then a transition in the fixed step order when the verdict changed, else just the band.
/// </summary>
private void RecomputeWarm()
{
var self = _localNode!.Value;
var entry = _lastSnapshot!.Nodes.FirstOrDefault(n => n.NodeId == self);
var now = _utcNow();
var memberState = SafeSelfStatus();
var listening = EndpointListening();
bool dbReachable;
bool stale;
if (_dbLess || _dbHealthProbe is null || _lastDbHealth is null)
{
dbReachable = true;
stale = entry is null || (now - entry.AsOfUtc) > _staleWindow;
}
else
{
dbReachable = _lastDbHealth.Reachable;
stale = !_lastDbHealth.Reachable
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|| entry is null || (now - entry.AsOfUtc) > _staleWindow;
}
var verdict = EvaluateWarmVerdict(self, memberState, listening);
TrackDisagreement(verdict, now);
var detached = entry is null || entry.Role == RedundancyRole.Detached;
var inputs = new NodeHealthInputs(memberState, dbReachable, OpcUaProbeOk(), stale, IsDriverPrimary: false, listening);
var target = detached ? WarmRole.Standby : verdict.Role;
var level = detached ? (byte)0 : ServiceLevelCalculator.ComputeWarm(inputs, target == WarmRole.Active);
ApplyWarmTransition(target, level, verdict.Reason);
}
private WarmVerdict EvaluateWarmVerdict(NodeId self, MemberStatus memberState, bool listening)
{
NodeId? oldest = null, oldestReachable = null;
var primaryUnreachable = false;
var snapshotPrimary = _acceptedSnapshot?.Nodes.FirstOrDefault(n => n.Role == RedundancyRole.Primary)?.NodeId;
try
{
var state = _cluster.State;
var unreachable = state.Unreachable.ToHashSet();
oldest = WarmRoleAuthority.OldestUpMember(state.Members, "driver");
oldestReachable = WarmRoleAuthority.OldestUpMember(state.Members, "driver", unreachable);
primaryUnreachable = snapshotPrimary is { } sp && unreachable.Any(m => WarmRoleAuthority.ToNodeId(m) == sp);
}
catch (Exception ex)
{
_log.Debug(ex, "OpcUaPublish: cluster state unavailable for the Warm verdict; treating membership as unknown");
}
return WarmRoleAuthority.Evaluate(new WarmRoleInputs(
self, memberState == MemberStatus.Up, listening, oldest, oldestReachable,
HasAcceptedSnapshot: _acceptedSnapshot is not null, snapshotPrimary, primaryUnreachable,
_redundancy.PromoteWhilePrimaryUnreachable));
}
private void TrackDisagreement(WarmVerdict verdict, DateTime now)
{
if (!verdict.Disagreement)
{
_disagreeSinceUtc = null;
_disagreementWarned = false;
return;
}
_disagreeSinceUtc ??= now;
if (_disagreementWarned || now - _disagreeSinceUtc.Value <= WarmDisagreementWarning) return;
_disagreementWarned = true;
_log.Warning(
"OpcUaPublish: Warm redundancy — cluster membership and the redundancy snapshot (revision {Revision}) have disagreed about this node being Primary for more than {Seconds}s; the node stays {Role}",
_revisionGate.LastAccepted, WarmDisagreementWarning.TotalSeconds, _warmRole);
}
/// <summary>
/// Moves the data plane to <paramref name="target"/> in the spec §4.7 order and publishes
/// <paramref name="level"/>. Invariant: an Active-band ServiceLevel is never visible while the projection is
/// closed — a promotion publishes the level LAST and, if that publish fails, retries it; a demotion publishes
/// the Standby level FIRST and degrades the data whether or not that publish succeeded (failing safe).
/// </summary>
private void ApplyWarmTransition(WarmRole target, byte level, string reason)
{
if (target == _warmRole)
{
if (!TryPublishServiceLevel(level) && target == WarmRole.Active) ScheduleWarmRetry();
else if (target == WarmRole.Active) _promotionLevelPending = false;
return;
}
var projection = _standbyProjection;
if (target == WarmRole.Active)
{
projection?.SetDataProjection(false); // 1. open the read projection
projection?.QueueCurrentToLiveItems(false); // 2. current Good value to every live data item
projection?.SetStandbyDelivery(false); // 3. first deliveries decided Active
projection?.SetAlarmSuppression(false); // 4. new alarm events delivered
projection?.RefreshAllEventItems(); // 5. promotion refresh on every live event item
_warmRole = WarmRole.Active;
if (TryPublishServiceLevel(level)) // 6. Active ServiceLevel, last
{
_promotionLevelPending = false;
}
else
{
_promotionLevelPending = true;
ScheduleWarmRetry();
}
}
else
{
TryPublishServiceLevel(level); // 1. Standby ServiceLevel, first (continue on failure)
projection?.SetStandbyDelivery(true); // 2. the D4 boundary
projection?.SetAlarmSuppression(true); // 3. suppress new alarm events
projection?.SetDataProjection(true); // 4. close the read projection
projection?.QueueCurrentToLiveItems(true); // 5. BadOutOfService to every live data item
_warmRole = WarmRole.Standby;
_promotionLevelPending = false;
}
OtOpcUaTelemetry.SetWarmRole(_warmRole == WarmRole.Active ? 1 : 0);
OtOpcUaTelemetry.WarmTransition.Add(1,
new KeyValuePair<string, object?>("direction", target == WarmRole.Active ? "promote" : "demote"),
new KeyValuePair<string, object?>("reason", reason));
_log.Info(
"OpcUaPublish: Warm redundancy {Direction} to {Role} (reason={Reason}, revision={Revision}, ServiceLevel={Level})",
target == WarmRole.Active ? "promotion" : "demotion", _warmRole, reason, _revisionGate.LastAccepted, level);
}
/// <summary>
/// Gitea #717 spec §3 deploy-time cross-check: under <c>None</c> or <c>Warm</c>, an artifact whose cluster
/// <c>RedundancyMode</c> disagrees with the local mode (<c>Hot</c> included, owner answer Q5) logs an Error and
/// counts <c>otopcua.redundancy.mode_mismatch</c>. The node keeps its local mode — the only value available
/// before SDK start. <c>Unset</c> has opted out and is not checked.
/// </summary>
/// <param name="artifact">The applied artifact bytes.</param>
private void CheckRedundancyMode(byte[] artifact)
{
if (_redundancy.Mode == OpcUaRedundancyMode.Unset || _localNode is null) return;
string? clusterMode;
try
{
clusterMode = DeploymentArtifact.ReadClusterRedundancyMode(artifact, _localNode.Value.Value);
}
catch (Exception ex)
{
_log.Debug(ex, "OpcUaPublish: could not read the artifact's cluster RedundancyMode");
return;
}
if (clusterMode is null) return;
if (string.Equals(clusterMode, _redundancy.Mode.ToString(), StringComparison.OrdinalIgnoreCase)) return;
OtOpcUaTelemetry.RedundancyModeMismatch.Add(1,
new KeyValuePair<string, object?>("local", _redundancy.Mode.ToString()),
new KeyValuePair<string, object?>("cluster", clusterMode));
_log.Error(
"OpcUaPublish: the deployed ServerCluster.RedundancyMode for this node's cluster is {ClusterMode} but OpcUa:Redundancy:Mode is {LocalMode}; the node keeps running as {LocalMode} (local configuration is the authority — fix whichever is wrong)",
clusterMode, _redundancy.Mode, _redundancy.Mode);
}
private void ScheduleWarmRetry() => Timers.StartSingleTimer("warm-retry", WarmRetry.Instance, TimeSpan.FromSeconds(1));
/// <summary>The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200,
/// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
@@ -0,0 +1,109 @@
using Akka.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
/// <summary>The Warm role (Gitea #717 spec §4.1).</summary>
public enum WarmRole
{
/// <summary>Degraded standby: projected data, suppressed new alarm events, Degraded-band ServiceLevel.</summary>
Standby,
/// <summary>Serves data and alarms; Healthy-band ServiceLevel.</summary>
Active,
}
/// <summary>The facts the Warm verdict is a pure function of.</summary>
/// <param name="Self">This node's canonical <c>host:port</c> id.</param>
/// <param name="SelfUp">This node's own member status is Up.</param>
/// <param name="EndpointListening">This node's OPC UA endpoint did not fail to start (#762).</param>
/// <param name="OldestUpDriver">The oldest Up <c>driver</c> member in this node's OWN cluster state (the singleton
/// placement rule), or null when there is none.</param>
/// <param name="OldestReachableUpDriver">The same, ignoring members this node sees as unreachable.</param>
/// <param name="HasAcceptedSnapshot">A revisioned snapshot has been accepted (spec §4.2). Unrevisioned snapshots
/// never count.</param>
/// <param name="SnapshotPrimary">The Primary named by the latest accepted revisioned snapshot, or null.</param>
/// <param name="SnapshotPrimaryUnreachable">This node sees <paramref name="SnapshotPrimary"/> as unreachable.</param>
/// <param name="PromoteWhilePrimaryUnreachable">The owner-answer-Q2 option (default on).</param>
public readonly record struct WarmRoleInputs(
NodeId Self,
bool SelfUp,
bool EndpointListening,
NodeId? OldestUpDriver,
NodeId? OldestReachableUpDriver,
bool HasAcceptedSnapshot,
NodeId? SnapshotPrimary,
bool SnapshotPrimaryUnreachable,
bool PromoteWhilePrimaryUnreachable);
/// <summary>The verdict and why.</summary>
/// <param name="Role">Active or Standby.</param>
/// <param name="Reason">A short reason for the transition metric/log: <c>boot</c>, <c>endpoint</c>,
/// <c>membership</c>, <c>snapshot</c>, <c>unreachable-primary</c> or <c>agreed</c>.</param>
/// <param name="Disagreement">Membership (2) and snapshot (3) disagree about whether this node is Primary.</param>
public readonly record struct WarmVerdict(WarmRole Role, string Reason, bool Disagreement);
/// <summary>
/// The Warm role verdict (Gitea #717 spec §4.1), a pure function so every row is unit-testable. Active iff the
/// node is Up and listening, its own cluster state names it the oldest Up driver member, and the latest accepted
/// revisioned snapshot names it Primary. Anything else — no snapshot yet, an unrevisioned snapshot, a
/// disagreement — is Standby. With <see cref="WarmRoleInputs.PromoteWhilePrimaryUnreachable"/> the survivor of a
/// crashed Primary goes Active once that Primary is unreachable, before <c>auto-down</c> removes it.
/// </summary>
/// <remarks>
/// This is a separate output from <c>PrimaryGatePolicy</c>, whose lone-node allow is unchanged for writes and
/// acks (D7).
/// </remarks>
public static class WarmRoleAuthority
{
/// <summary>Evaluates the verdict.</summary>
/// <param name="i">The inputs.</param>
/// <returns>The verdict.</returns>
public static WarmVerdict Evaluate(in WarmRoleInputs i)
{
if (!i.SelfUp) return new(WarmRole.Standby, "membership", false);
if (!i.EndpointListening) return new(WarmRole.Standby, "endpoint", false);
if (!i.HasAcceptedSnapshot) return new(WarmRole.Standby, "boot", false);
var membershipSaysSelf = i.OldestUpDriver == i.Self;
var snapshotSaysSelf = i.SnapshotPrimary == i.Self;
var disagreement = membershipSaysSelf != snapshotSaysSelf;
if (membershipSaysSelf && snapshotSaysSelf) return new(WarmRole.Active, "agreed", false);
// Owner answer Q2: a SIGKILLed Primary stays an Up-but-unreachable member until auto-down removes it, and the
// singleton usually died with it. Evaluate (2) and (3) with unreachable members excluded.
if (i.PromoteWhilePrimaryUnreachable
&& i.OldestReachableUpDriver == i.Self
&& (snapshotSaysSelf || (i.SnapshotPrimary is not null && i.SnapshotPrimaryUnreachable)))
{
return new(WarmRole.Active, "unreachable-primary", disagreement);
}
return new(WarmRole.Standby, membershipSaysSelf ? "snapshot" : "membership", disagreement);
}
/// <summary>The oldest Up member carrying <paramref name="role"/> (the singleton placement rule), optionally
/// ignoring <paramref name="unreachable"/> members.</summary>
/// <param name="members">The current members.</param>
/// <param name="role">The role.</param>
/// <param name="unreachable">Members to ignore, or null.</param>
/// <returns>Its canonical node id, or null.</returns>
public static NodeId? OldestUpMember(IEnumerable<Member> members, string role, IReadOnlySet<Member>? unreachable = null)
{
Member? oldest = null;
foreach (var m in members)
{
if (m.Status != MemberStatus.Up || !m.HasRole(role)) continue;
if (unreachable is not null && unreachable.Contains(m)) continue;
if (oldest is null || m.IsOlderThan(oldest)) oldest = m;
}
return oldest is null ? null : ToNodeId(oldest);
}
/// <summary>The canonical <c>host:port</c> id of a member (the RedundancyStateActor format).</summary>
/// <param name="m">The member.</param>
/// <returns>The node id.</returns>
public static NodeId ToNodeId(Member m) => NodeId.Parse($"{m.Address.Host ?? string.Empty}:{m.Address.Port ?? 0}");
}
@@ -566,6 +566,9 @@ public static class ServiceCollectionExtensions
// Optional: a harness that builds this graph without the hosted service registers none, and both
// consumers then behave as if the endpoint were listening (the pre-#762 behaviour).
var endpointState = resolver.GetService<IOpcUaEndpointState>();
// Gitea #717: OpcUa:Redundancy (registered by the Host from the validated OpcUa options). Absent in harnesses ⇒
// Unset ⇒ today's behaviour. The standby projection is late-bound like the ServiceLevel publisher.
var redundancy = resolver.GetService<WarmRedundancySettings>() ?? WarmRedundancySettings.Unset;
var publishActor = system.ActorOf(
OpcUaPublishActor.Props(
sink: addressSpaceSink,
@@ -576,7 +579,9 @@ public static class ServiceCollectionExtensions
dbHealthProbe: dbHealth,
dbLess: dbFactory is null,
dataPlaneAuthority: dataPlaneAuthority,
endpointState: endpointState),
endpointState: endpointState,
redundancy: redundancy,
standbyProjection: redundancy.IsWarm ? resolver.GetService<DeferredStandbyProjection>() : null),
OpcUaPublishActorName);
registry.Register<OpcUaPublishActorKey>(publishActor);
@@ -670,7 +675,9 @@ public static class ServiceCollectionExtensions
// lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
// the InProcess default, when the seam is not listed, and in test graphs that register no provider.
scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm),
// Gitea #717 spec §4.2: drop older redundancy snapshots under Warm only.
enforceSnapshotOrdering: redundancy.IsWarm),
DriverHostActorName);
registry.Register<DriverHostActorKey>(driverHost);
@@ -0,0 +1,154 @@
using Akka.Actor;
using Akka.Cluster;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// Gitea #717 spec §4.2 (ordered snapshots) and §4.3 (Warm bands): the revision the singleton stamps, its order,
/// the subscriber gate, the wire shape of the extended message, and the Warm ServiceLevel truth table.
/// </summary>
public sealed class WarmRedundancyRevisionTests : ControlPlaneActorTestBase
{
[Fact]
public void The_singleton_stamps_an_increasing_revision_on_every_publish_heartbeats_included()
{
var probe = CreateTestProbe();
Sys.ActorOf(RedundancyStateActor.Props(msg => probe.Ref.Tell(msg), TimeSpan.FromMilliseconds(200)));
var first = probe.ExpectMsg<RedundancyStateChanged>(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);
var second = probe.ExpectMsg<RedundancyStateChanged>(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);
first.Revision.ShouldNotBeNull();
second.Revision.ShouldNotBeNull();
second.Revision!.CompareTo(first.Revision).ShouldBeGreaterThan(0);
second.Revision.Sequence.ShouldBe(first.Revision!.Sequence + 1);
first.Revision.PublisherUpNumber.ShouldBeGreaterThan(0L, "the publisher's Akka up-number");
}
/// <summary>Akka.NET keeps <c>Member.UpNumber</c> internal; an upgrade that renames it must fail here, not degrade
/// every revision to up-number 0 silently.</summary>
[Fact]
public void Akka_member_up_number_is_readable()
{
AwaitCondition(() => Akka.Cluster.Cluster.Get(Sys).SelfMember.Status == MemberStatus.Up, TimeSpan.FromSeconds(10),
cancellationToken: TestContext.Current.CancellationToken);
RedundancyStateActor.ReadUpNumber(Akka.Cluster.Cluster.Get(Sys).SelfMember).ShouldNotBeNull().ShouldBeGreaterThan(0);
}
[Fact]
public void A_singleton_restarted_in_the_same_process_outranks_its_predecessor()
{
var a = RedundancyStateActor.NextIncarnationTick();
var b = RedundancyStateActor.NextIncarnationTick();
b.ShouldBeGreaterThan(a, "strictly increasing even within one tick");
new SnapshotRevision(1, b, 1).CompareTo(new SnapshotRevision(1, a, 500)).ShouldBeGreaterThan(0);
}
[Fact]
public void Revisions_order_by_up_number_then_incarnation_then_sequence()
{
var baseline = new SnapshotRevision(2, 100, 50);
new SnapshotRevision(3, 1, 1).CompareTo(baseline).ShouldBeGreaterThan(0, "a handover to a younger member outranks");
new SnapshotRevision(1, 999, 999).CompareTo(baseline).ShouldBeLessThan(0, "a departed older member's late publish loses");
new SnapshotRevision(2, 101, 1).CompareTo(baseline).ShouldBeGreaterThan(0);
new SnapshotRevision(2, 100, 51).CompareTo(baseline).ShouldBeGreaterThan(0);
new SnapshotRevision(2, 100, 50).CompareTo(baseline).ShouldBe(0);
}
[Fact]
public void The_gate_accepts_only_newer_revisions_and_never_advances_on_an_unrevisioned_snapshot()
{
var gate = new SnapshotRevisionGate();
gate.Offer(Snap(null)).ShouldBe(SnapshotRejection.Unrevisioned);
gate.LastAccepted.ShouldBeNull();
gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.None);
gate.Offer(Snap(new(1, 9, 9))).ShouldBe(SnapshotRejection.OlderRevision, "an old incarnation's late heartbeat");
gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.OlderRevision, "a duplicate");
gate.Offer(Snap(new(2, 1, 6))).ShouldBe(SnapshotRejection.None);
gate.LastAccepted.ShouldBe(new SnapshotRevision(2, 1, 6));
}
[Fact]
public void The_extended_message_round_trips_and_a_pre_717_payload_decodes_with_no_revision()
{
var msg = Snap(new SnapshotRevision(3, 42, 7));
var serializer = Sys.Serialization.FindSerializerFor(msg);
var manifest = Akka.Serialization.Serialization.ManifestFor(serializer, msg);
var round = (RedundancyStateChanged)Sys.Serialization.Deserialize(serializer.ToBinary(msg), serializer.Identifier, manifest);
round.Revision.ShouldBe(msg.Revision);
round.Nodes.Single().NodeId.ShouldBe(msg.Nodes.Single().NodeId);
// A pre-#717 publisher's payload: the same message without the field.
var legacy = Snap(null);
var legacyBytes = serializer.ToBinary(legacy);
var text = System.Text.Encoding.UTF8.GetString(legacyBytes);
var stripped = System.Text.Encoding.UTF8.GetBytes(text.Replace(",\"Revision\":null", string.Empty, StringComparison.Ordinal));
var decoded = (RedundancyStateChanged)Sys.Serialization.Deserialize(stripped, serializer.Identifier, manifest);
decoded.Revision.ShouldBeNull();
decoded.Nodes.Count.ShouldBe(1);
}
private static RedundancyStateChanged Snap(SnapshotRevision? revision) =>
new(new[] { new NodeRedundancyState(NodeId.Parse("a:1"), RedundancyRole.Primary, true, true, DateTime.UtcNow) },
CorrelationId.NewId(), revision);
}
/// <summary>Gitea #717 spec §4.3: the Warm ServiceLevel bands (pure; no actor system).</summary>
public sealed class WarmServiceLevelBandTests
{
public static IEnumerable<object[]> AllInputs()
{
foreach (var member in new[] { MemberStatus.Up, MemberStatus.Joining, MemberStatus.Leaving, MemberStatus.Down })
foreach (var db in new[] { true, false })
foreach (var probe in new[] { true, false })
foreach (var stale in new[] { true, false })
foreach (var listening in new[] { true, false })
yield return [member, db, probe, stale, listening];
}
[Theory]
[MemberData(nameof(AllInputs))]
public void Every_standby_level_is_below_every_active_level_that_serves_data(
MemberStatus member, bool db, bool probe, bool stale, bool listening)
{
var h = new NodeHealthInputs(member, db, probe, stale, IsDriverPrimary: false, listening);
var standby = ServiceLevelCalculator.ComputeWarm(h, isActive: false);
var active = ServiceLevelCalculator.ComputeWarm(h, isActive: true);
standby.ShouldBeLessThanOrEqualTo(ServiceLevelCalculator.WarmStandbyHealthy);
if (active > 0) active.ShouldBeGreaterThanOrEqualTo(ServiceLevelCalculator.WarmActiveCritical);
if (active > 0) standby.ShouldBeLessThan(active);
}
[Theory]
[InlineData(true, true, false, true, 250)]
[InlineData(true, true, true, true, 200)]
[InlineData(false, true, true, true, 100)]
[InlineData(false, true, false, true, 100)]
[InlineData(true, false, false, true, 100)]
[InlineData(true, true, false, false, 90)]
[InlineData(true, true, true, false, 50)]
[InlineData(false, true, false, false, 50)]
[InlineData(true, false, false, false, 50)]
public void Warm_bands_match_the_spec_table(bool db, bool probe, bool stale, bool active, int expected)
{
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, db, probe, stale, IsDriverPrimary: false), active)
.ShouldBe((byte)expected);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Down_or_a_failed_endpoint_is_zero_in_either_role(bool active)
{
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Down, true, true, false, false), active).ShouldBe((byte)0);
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, true, true, false, false, EndpointListening: false), active)
.ShouldBe((byte)0);
}
}
@@ -132,3 +132,38 @@ public sealed class OpcUaApplicationHostOptionsValidatorTests
if (!valid) result.Failures!.ShouldContain(f => f.Contains("OpcUa:Discovery"));
}
}
/// <summary>Gitea #717 spec §3: the redundancy rules the startup validator and the driver-role guard enforce.</summary>
public sealed class OpcUaRedundancyValidationTests
{
private static readonly OpcUaApplicationHostOptionsValidator Sut = new();
[Theory]
[InlineData("Unset", new string[0], true)]
[InlineData("None", new string[0], true)]
[InlineData("None", new[] { "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:peer" }, true)]
[InlineData("Warm", new string[0], false)]
[InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:OtOpcUa" }, false)]
[InlineData("Hot", new[] { "urn:peer" }, false)]
[InlineData("Bogus", new string[0], false)]
public void Redundancy_matrix(string mode, string[] peers, bool valid)
{
var result = Sut.Validate(null, new OpcUaApplicationHostOptions
{
Redundancy = new OpcUaRedundancyOptions { Mode = mode },
PeerApplicationUris = peers.ToList(),
});
result.Succeeded.ShouldBe(valid, result.FailureMessage);
}
[Fact]
public void Warm_on_a_node_without_the_driver_role_fails_start()
{
Should.Throw<InvalidOperationException>(() => OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: false));
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: true);
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(null, hasDriver: false);
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("None", hasDriver: false);
}
}
@@ -44,6 +44,9 @@ internal sealed class ServerEventRig : IAsyncDisposable
public string EndpointUrl => $"opc.tcp://127.0.0.1:{_port}/OtOpcUa";
/// <summary>The configured <c>OpcUa:ApplicationUri</c> (the effective one: the rig loads no XML).</summary>
public string ApplicationUri => _hostOptions.ApplicationUri;
public OtOpcUaNodeManager Nm => _server.NodeManager!;
internal OtOpcUaSdkServer Server => _server;
@@ -57,7 +60,12 @@ internal sealed class ServerEventRig : IAsyncDisposable
public SdkAddressSpaceSink Sink { get; private set; } = null!;
public static async Task<ServerEventRig> StartAsync(string name, bool encrypted, CancellationToken ct)
public static Task<ServerEventRig> StartAsync(string name, bool encrypted, CancellationToken ct) =>
StartAsync(name, encrypted, ct, configure: null);
/// <summary>Starts the rig, letting the caller adjust the host options first (Gitea #717: redundancy mode, peers).</summary>
public static async Task<ServerEventRig> StartAsync(
string name, bool encrypted, CancellationToken ct, Action<OpcUaApplicationHostOptions>? configure)
{
var rig = new ServerEventRig(Path.Combine(Path.GetTempPath(), $"otopcua-{name}-{Guid.NewGuid():N}"));
rig._hostOptions = new OpcUaApplicationHostOptions
@@ -71,6 +79,7 @@ internal sealed class ServerEventRig : IAsyncDisposable
: [OpcUaSecurityProfile.None],
AutoAcceptUntrustedClientCertificates = true,
};
configure?.Invoke(rig._hostOptions);
(rig._host, rig._server) = await OpcUaHostStart.StartAsync(
port =>
{
@@ -0,0 +1,616 @@
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Gitea #717 over the wire, against the real <see cref="OtOpcUaSdkServer"/> and node manager: the redundancy
/// metadata per mode (T2), the Standby read projection and write refusal (T6), the D4 first-delivery decision record
/// across the three demotion timings T0 established (T7), the live-item queue on promotion/demotion (T8), and the
/// alarm suppression, refusal and promotion refresh (T9). Transitions are driven through the server's own
/// <see cref="IStandbyProjection"/> in the spec §4.7 step order; the actor that owns that order has its own tests.
/// </summary>
[Trait("Category", "Integration")]
public sealed class WarmRedundancyWireTests
{
private const string Peer = "urn:OtOpcUa:peer-717";
private const NodePermissions All =
NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead
| NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve
| NodePermissions.MethodCall | NodePermissions.WriteOperate;
private static Task<ServerEventRig> StartModeAsync(string name, string mode, CancellationToken ct) =>
ServerEventRig.StartAsync(name, encrypted: false, ct, o =>
{
o.Redundancy = new OpcUaRedundancyOptions { Mode = mode };
if (mode == "Warm") o.PeerApplicationUris = [Peer];
});
/// <summary>Spec §4.7 promotion, in order (the actor's sequence, minus the ServiceLevel publish).</summary>
internal static void Promote(IStandbyProjection p)
{
p.SetDataProjection(false);
p.QueueCurrentToLiveItems(false);
p.SetStandbyDelivery(false);
p.SetAlarmSuppression(false);
p.RefreshAllEventItems();
}
/// <summary>Spec §4.7 demotion, in order (after the Standby ServiceLevel publish).</summary>
internal static void Demote(IStandbyProjection p)
{
p.SetStandbyDelivery(true);
p.SetAlarmSuppression(true);
p.SetDataProjection(true);
p.QueueCurrentToLiveItems(true);
}
private static CancellationToken Deadline(int seconds = 90)
{
var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
cts.CancelAfter(TimeSpan.FromSeconds(seconds));
return cts.Token;
}
private static RequestHeader H() => new() { Timestamp = DateTimeUtc.Now, TimeoutHint = 10_000 };
private static NodeId Raw(ISession session, string id) =>
new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
private static async Task<DataValue> ReadAsync(ISession session, NodeId node, CancellationToken ct, uint attribute = Attributes.Value,
string? indexRange = null)
{
var r = await ((SessionClient)session).ReadAsync(H(), 0, TimestampsToReturn.Both,
new[] { new ReadValueId { NodeId = node, AttributeId = attribute, IndexRange = indexRange } }, ct);
return r.Results[0];
}
// ------------------------------------------------------------------ T2 metadata
[Fact]
public async Task Warm_advertises_non_transparent_metadata_self_first_and_boots_at_ServiceLevel_1()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Meta", "Warm", ct);
using var session = await rig.ConnectAsync(anonymous: true, ct);
Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
.ShouldBe((int)RedundancySupport.Warm);
var uris = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
uris.StatusCode.ShouldBe(StatusCodes.Good);
((string[])uris.ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.WarmBootServiceLevel);
var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
{
new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
}, ct);
browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.NonTransparentRedundancyType));
var write = await ((SessionClient)session).WriteAsync(H(), new[]
{
new WriteValue { NodeId = VariableIds.Server_ServerRedundancy_ServerUriArray, AttributeId = Attributes.Value,
Value = new DataValue(new Variant(new[] { "x" })) },
}, ct);
StatusCode.IsBad(write.Results[0]).ShouldBeTrue($"ServerUriArray is read-only, got {write.Results[0]}");
}
[Fact]
public async Task None_advertises_RedundancySupport_None_and_ServerUriArray_is_absent_even_by_NodeId()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717None", "None", ct);
using var session = await rig.ConnectAsync(anonymous: true, ct);
Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
.ShouldBe((int)RedundancySupport.None);
(await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct)).StatusCode
.ShouldBe(StatusCodes.BadNodeIdUnknown, "T0a: the SDK's orphan i=11314 is deleted under None");
var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
{
new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
}, ct);
browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.ServerRedundancyType));
((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri });
}
/// <summary>The default-off pin: an Unset server is today's server — SDK-default RedundancySupport, the SDK's own
/// (orphan) i=11314, ServerRedundancyType, the #885 boot level 0, Good data with no projection, and no Warm hooks.</summary>
[Fact]
public async Task Unset_is_todays_server_exactly()
{
var ct = Deadline();
await using var rig = await ServerEventRig.StartAsync("Warm717Unset", encrypted: false, ct);
rig.Server.RedundancyMode.ShouldBe(OpcUaRedundancyMode.Unset);
rig.Server.CreateStandbyProjection().ShouldBeNull();
rig.Nm.Standby.WarmMode.ShouldBeFalse();
var id = rig.EnsureRawVariable("u", "Double", writable: false);
rig.Sink.WriteValue(id, 4.5, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var support = await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct);
support.StatusCode.ShouldBe(StatusCodes.Good);
Convert.ToInt32(support.ClrValue()).ShouldBe((int)RedundancySupport.None, "the SDK default, never written by us");
var orphan = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
orphan.StatusCode.ShouldBe(StatusCodes.Good, "Unset leaves the SDK's orphan i=11314 exactly as the SDK builds it");
Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel);
var value = await ReadAsync(session, Raw(session, id), ct);
value.StatusCode.ShouldBe(StatusCodes.Good);
value.ClrValue().ShouldBe(4.5);
}
// ------------------------------------------------------------------ T6 read projection + write refusal
[Fact]
public async Task Standby_projects_ordinary_tag_reads_but_not_metadata_and_promotion_reads_the_latest_stored_value()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Read", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection().ShouldNotBeNull();
var raw = rig.EnsureRawVariable("t1", "Double", writable: false);
rig.Sink.EnsureVariable("EqA/t1ref", null, "t1ref", "Double", writable: false, AddressSpaceRealm.Uns);
var arr = $"{RawDeviceFolder}/arr";
rig.Sink.EnsureVariable(arr, RawDeviceFolder, "arr", "Int32", writable: false, AddressSpaceRealm.Raw, isArray: true);
rig.EnsureEquipment();
rig.Sink.EnsureVariable("EqA/vt", "EqA", "vt", "Double", writable: false, AddressSpaceRealm.Uns);
rig.MaterialiseScripted();
var ts = DateTime.UtcNow;
rig.Sink.WriteValue(raw, 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
rig.Sink.WriteValue("EqA/t1ref", 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
rig.Sink.WriteValue("EqA/vt", 2.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
rig.Sink.WriteValue(arr, new[] { 1, 2, 3 }, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
using var session = await rig.ConnectAsync(anonymous: true, ct);
foreach (var node in new[] { Raw(session, raw), UnsNodeId(session, "EqA/t1ref"), UnsNodeId(session, "EqA/vt") })
{
var dv = await ReadAsync(session, node, ct);
dv.StatusCode.ShouldBe(StatusCodes.BadOutOfService, node.ToString());
dv.WrappedValue.IsNull.ShouldBeTrue();
(dv.ServerTimestamp != DateTimeUtc.MinValue).ShouldBeTrue();
}
(await ReadAsync(session, Raw(session, arr), ct, indexRange: "1")).StatusCode.ShouldBe(StatusCodes.BadOutOfService);
(await ReadAsync(session, Raw(session, raw), ct, Attributes.DisplayName)).StatusCode.ShouldBe(StatusCodes.Good);
(await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).StatusCode.ShouldBe(StatusCodes.Good);
(await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).StatusCode.ShouldBe(StatusCodes.Good);
var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
(await ReadAsync(session, severity, ct)).StatusCode.ShouldBe(StatusCodes.Good, "a condition child is not an ordinary tag");
// Acquisition continues while projected; promotion reads the latest stored value with no new driver publish.
rig.Sink.WriteValue(raw, 9.25, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
var after = await ReadAsync(session, Raw(session, raw), ct);
after.StatusCode.ShouldBe(StatusCodes.Good);
after.ClrValue().ShouldBe(9.25);
var ranged = await ReadAsync(session, Raw(session, arr), ct, indexRange: "1");
ranged.StatusCode.ShouldBe(StatusCodes.Good);
((int[])ranged.ClrValue()!).ShouldBe(new[] { 2 }, "an IndexRange read reaches the stored array once promoted");
}
[Fact]
public async Task Standby_projection_never_leaks_an_ACL_denied_node()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Acl", "Warm", ct);
var raw = rig.EnsureRawVariable("secret", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Enforce(true, [Key()]); // anonymous holds no grant
using var session = await rig.ConnectAsync(anonymous: true, ct);
(await ReadAsync(session, Raw(session, raw), ct)).StatusCode.ShouldBe(StatusCodes.BadUserAccessDenied);
}
private sealed class RecordingWriteGateway : IOpcUaNodeWriteGateway
{
public int Count;
public Task<NodeWriteOutcome> WriteAsync(string nodeId, object? value, AddressSpaceRealm realm, CancellationToken ct)
{
Interlocked.Increment(ref Count);
return Task.FromResult(new NodeWriteOutcome(true, null));
}
}
[Theory]
[InlineData("Warm", true)]
[InlineData("Warm", false)]
[InlineData("Unset", true)]
public async Task A_write_on_a_Warm_standby_is_refused_BadInvalidState_and_never_reaches_the_gateway(string mode, bool standby)
{
var ct = Deadline();
await using var rig = await StartModeAsync($"Warm717Write{mode}{standby}", mode, ct);
var gateway = new RecordingWriteGateway();
rig.Nm.NodeWriteGateway = gateway;
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
var raw = rig.EnsureRawVariable("w", "Double", writable: true);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
if (mode == "Warm" && !standby) Promote(rig.Server.CreateStandbyProjection()!);
// Unset with "standby": a node that is not the Primary takes today's optimistic path (no gate wired here).
using var session = await rig.ConnectAsync(anonymous: false, ct);
var result = await ((SessionClient)session).WriteAsync(H(), new[]
{
new WriteValue { NodeId = Raw(session, raw), AttributeId = Attributes.Value, Value = new DataValue(new Variant(2.0)) },
}, ct);
if (mode == "Warm" && standby)
{
result.Results[0].ShouldBe(StatusCodes.BadInvalidState);
await Task.Delay(200, ct);
gateway.Count.ShouldBe(0, "a refused write never reaches the driver");
}
else
{
result.Results[0].ShouldBe(StatusCodes.Good);
await RecordedEvents.WaitUntilAsync(() => Volatile.Read(ref gateway.Count) == 1);
}
}
// ------------------------------------------------------------------ T9 Part 9 method refusal
[Theory]
[InlineData("Warm", true, "Acknowledge")]
[InlineData("Warm", true, "OneShotShelve")]
[InlineData("Warm", false, "Acknowledge")]
[InlineData("Unset", true, "Acknowledge")]
public async Task A_Part9_method_on_a_Warm_standby_is_refused_BadInvalidState_before_any_state_change(
string mode, bool standby, string method)
{
var ct = Deadline();
await using var rig = await StartModeAsync($"Warm717Method{mode}{standby}{method}", mode, ct);
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseScripted();
var routed = 0;
rig.Nm.AlarmCommandRouter = _ => Interlocked.Increment(ref routed);
var projection = rig.Server.CreateStandbyProjection();
if (projection is not null) Promote(projection); // project the condition while Active so it has an EventId
rig.Project(native: false, ConditionMethodEventTests.PreState(method, native: false) with { Message = "base" });
if (projection is not null && standby) Demote(projection);
using var session = await rig.ConnectAsync(anonymous: false, ct);
var eventId = await ReadEventIdAsync(session, ct);
var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), method, eventId, ct);
if (mode == "Warm" && standby)
{
(await Should.ThrowAsync<ServiceResultException>(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState);
routed.ShouldBe(0, "nothing is routed to the engine");
var acked = await TranslateAsync(session, ConditionNodeId(session, native: false), "AckedState/Id", ct);
(await ReadAsync(session, acked, ct)).ClrValue().ShouldBe(false, "no condition state changed");
}
else
{
await call;
routed.ShouldBe(1);
}
}
private static async Task<byte[]> ReadEventIdAsync(ISession session, CancellationToken ct)
{
var eventId = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.EventId, ct);
return ((ByteString)(await ReadAsync(session, eventId, ct)).WrappedValue.AsBoxedObject()!).ToArray()!;
}
private static async Task<NodeId> TranslateAsync(ISession session, NodeId start, string path, CancellationToken ct)
{
var elements = path.Split('/').Select(name => new RelativePathElement
{
ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, TargetName = new QualifiedName(name),
}).ToArray();
var r = await ((SessionClient)session).TranslateBrowsePathsToNodeIdsAsync(H(), new[]
{
new BrowsePath { StartingNode = start, RelativePath = new RelativePath { Elements = elements } },
}, ct);
r.Results[0].StatusCode.ShouldBe(StatusCodes.Good, path);
return ExpandedNodeId.ToNodeId(r.Results[0].Targets[0].TargetId, session.NamespaceUris);
}
// ------------------------------------------------------------------ T9 alarm suppression + refresh
[Fact]
public async Task Standby_suppresses_new_alarm_events_on_every_item_kind_and_promotion_refreshes_them_all()
{
var ct = Deadline(120);
await using var rig = await StartModeAsync("Warm717Alarms", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseNative();
rig.MaterialiseScripted();
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
var folderItem = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA));
var deviceItem = RecordedEvents.AddEventItem(subscription,
new NodeId(RawDeviceFolder, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri)));
await subscription.ApplyChangesAsync(ct);
// While Standby (boot state): a new transition reaches no item, but the condition node is current.
rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "while-standby"));
// A condition raised and cleared (Retain=false) while Standby is not replayed by the promotion refresh.
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash"));
rig.Project(native: false, new AlarmConditionSnapshot(Active: false, Acknowledged: true, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash-cleared"));
await Task.Delay(1000, ct);
foreach (var item in new[] { serverItem, folderItem, deviceItem })
{
events.Has(item, "while-standby").ShouldBeFalse($"no alarm event on {item.StartNodeId} while Standby");
events.Has(item, "flash").ShouldBeFalse();
}
// D5: a client ConditionRefresh on the Standby returns Start, the retained condition, End.
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
events.Has(serverItem, "while-standby").ShouldBeTrue("the Standby's refresh carries the retained condition");
events.Has(serverItem, "flash-cleared").ShouldBeFalse("a non-retained condition is not replayed");
foreach (var item in new[] { folderItem, deviceItem })
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
var countsBefore = new[] { serverItem, folderItem, deviceItem }.ToDictionary(i => i, i => events.Of(i).Count);
Promote(projection);
foreach (var item in new[] { serverItem, folderItem, deviceItem })
{
await RecordedEvents.WaitUntilAsync(() => events.Of(item).Skip(countsBefore[item])
.Any(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType));
var stream = events.Of(item).Skip(countsBefore[item]).ToList();
var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
start.ShouldBeGreaterThanOrEqualTo(0, $"RefreshStart on {item.StartNodeId}");
var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
replayed.ShouldContain("while-standby", $"promotion refresh replays the retained condition on {item.StartNodeId}");
replayed.ShouldNotContain("flash-cleared");
}
// After promotion new events flow, and the Standby-stamped EventId acknowledges.
rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 800, Message: "after"));
await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "after") && events.Has(folderItem, "after") && events.Has(deviceItem, "after"));
await subscription.DeleteAsync(true, ct);
}
[Fact]
public async Task A_Standby_stamped_EventId_is_accepted_by_Acknowledge_after_promotion()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717AckAfter", "Warm", ct);
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseScripted();
var routed = new List<AlarmCommand>();
rig.Nm.AlarmCommandRouter = cmd => { lock (routed) routed.Add(cmd); };
rig.Project(native: false, ConditionMethodEventTests.PreState("Acknowledge", native: false) with { Message = "stamped" });
using var session = await rig.ConnectAsync(anonymous: false, ct);
var stamped = await ReadEventIdAsync(session, ct); // stamped while Standby, never delivered
Promote(rig.Server.CreateStandbyProjection()!);
await ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), "Acknowledge", stamped, ct);
lock (routed) routed.ShouldHaveSingleItem().Operation.ShouldBe("Acknowledge");
}
// ------------------------------------------------------------------ T8 live items on transition
[Fact]
public async Task Demotion_queues_one_BadOutOfService_then_silence_and_promotion_queues_the_current_value_through_a_deadband()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Live", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("live", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
await AddItemAsync(client, sub, Raw(session, raw), 2, 10, ct, new DataChangeFilter
{
Trigger = DataChangeTrigger.StatusValue, DeadbandType = (uint)DeadbandType.Absolute, DeadbandValue = 1000,
});
(await CollectAsync(client, ct, TimeSpan.FromSeconds(1))).Where(x => x.Handle is 1 or 2).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
Demote(projection);
rig.Sink.WriteValue(raw, 2.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 3.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
var demoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
demoted.Count(x => x.Handle == 1).ShouldBe(1, "exactly one notification per item, then silence");
demoted.Count(x => x.Handle == 2).ShouldBe(1);
demoted.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
rig.Sink.WriteValue(raw, 4.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
var promoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
promoted.ShouldContain(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0));
promoted.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0),
"the promotion value is queued ignoring the deadband");
}
// ------------------------------------------------------------------ T7 first-delivery decision record (D4)
/// <summary>Timing 1 (T0): Good samples queued in the item while Active, a client-handle change, then demotion.</summary>
[Fact]
public async Task Timing1_queued_good_samples_then_demotion_first_delivery_is_projected_and_republish_matches()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717T1", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("q", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.MaterialiseScripted();
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
var tagItem = await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct);
var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
await AddItemAsync(client, sub, severity, 3, 1, ct);
(await PublishDataAsync(client, ct)).NotificationMessage.NotificationData.Count.ShouldBeGreaterThan(0);
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
// Non-ordinary changes queued in the same window: i=2267 and a condition child (Severity).
new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
Microsoft.Extensions.Logging.Abstractions.NullLogger<SdkServiceLevelPublisher>.Instance).Publish(90);
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 650, Message: "child"));
await client.ModifyMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
{
new MonitoredItemModifyRequest { MonitoredItemId = tagItem,
RequestedParameters = new MonitoringParameters { ClientHandle = 99, SamplingInterval = 0, QueueSize = 10, DiscardOldest = true } },
}, ct);
DemoteDeliveryOnly(projection);
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
var msg = await PublishDataAsync(client, ct);
var changes = DataChanges(msg.NotificationMessage);
changes.Where(x => x.Handle is 1 or 99).ShouldNotBeEmpty();
changes.Where(x => x.Handle is 1 or 99).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService && x.Value.WrappedValue.IsNull,
"Good samples queued before demotion are first-delivered projected (D4)");
changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not an ordinary tag");
changes.ShouldContain(x => x.Handle == 3 && x.Value.StatusCode == StatusCodes.Good, "a condition child is not an ordinary tag");
var again = await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct);
DataChanges(again.NotificationMessage).Where(x => x.Handle is 1 or 99)
.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService, "Republish returns the recorded projected choice");
}
/// <summary>Timing 2 (T0): messages prebuilt while Active (one notification per message), then demotion.</summary>
[Fact]
public async Task Timing2_prebuilt_messages_then_demotion_next_publish_is_projected_and_unsent_republish_is_unavailable()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717T2", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("p", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 1, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
DataChanges((await PublishDataAsync(client, ct)).NotificationMessage).Count.ShouldBe(1);
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
rig.Sink.WriteValue(raw, 20.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 21.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 22.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
var a = await PublishDataAsync(client, ct); // builds several messages while Active; the first is delivered Good
DataChanges(a.NotificationMessage).Single().Value.StatusCode.ShouldBe(StatusCodes.Good);
DemoteDeliveryOnly(projection);
var next = a.NotificationMessage.SequenceNumber + 1;
(await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, next, ct).AsTask())).StatusCode
.ShouldBe(StatusCodes.BadMessageNotAvailable, "2.0.1: a prebuilt, never-sent message cannot be first-delivered by Republish");
var b = await PublishDataAsync(client, ct);
b.NotificationMessage.SequenceNumber.ShouldBe(next);
DataChanges(b.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService,
"a message built while Active but first delivered after demotion is projected");
var bRe = await client.RepublishAsync(H(), sub, next, ct);
DataChanges(bRe.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
}
/// <summary>Timing 3 (T0): Good delivered while Active, then demotion — Republish returns the original Good; after an
/// acknowledge it is gone.</summary>
[Fact]
public async Task Timing3_good_delivered_while_active_republishes_good_after_demotion_and_ack_drops_it()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717T3", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("d", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
var first = await PublishDataAsync(client, ct);
DataChanges(first.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
Demote(projection);
var again = await client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct);
DataChanges(again.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good,
"Republish of a message first delivered while Active returns the original (D4 boundary)");
await client.PublishAsync(H(), new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = first.NotificationMessage.SequenceNumber } }, ct);
(await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct).AsTask()))
.StatusCode.ShouldBe(StatusCodes.BadMessageNotAvailable);
}
/// <summary>The D4 boundary alone (spec §4.7 demotion step 2), without the projection that follows it, so the
/// decision record is what the test observes.</summary>
private static void DemoteDeliveryOnly(IStandbyProjection p) => p.SetStandbyDelivery(true);
// ------------------------------------------------------------------ raw client helpers
private static async Task<uint> AddItemAsync(SessionClient client, uint sub, NodeId node, uint handle, uint queue, CancellationToken ct,
MonitoringFilter? filter = null)
{
var r = await client.CreateMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
{
new MonitoredItemCreateRequest
{
ItemToMonitor = new ReadValueId { NodeId = node, AttributeId = Attributes.Value },
MonitoringMode = MonitoringMode.Reporting,
RequestedParameters = new MonitoringParameters
{
ClientHandle = handle, SamplingInterval = 0, QueueSize = queue, DiscardOldest = true,
Filter = filter is null ? ExtensionObject.Null : new ExtensionObject(filter),
},
},
}, ct);
StatusCode.IsGood(r.Results[0].StatusCode).ShouldBeTrue($"create {node}: {r.Results[0].StatusCode}");
return r.Results[0].MonitoredItemId;
}
internal static List<(uint Handle, DataValue Value)> DataChanges(NotificationMessage m)
{
var list = new List<(uint, DataValue)>();
for (var i = 0; i < m.NotificationData.Count; i++)
{
if (m.NotificationData[i].TryGetValue<DataChangeNotification>(out var d))
for (var j = 0; j < d.MonitoredItems.Count; j++) list.Add((d.MonitoredItems[j].ClientHandle, d.MonitoredItems[j].Value));
}
return list;
}
private static async Task<PublishResponse> PublishDataAsync(SessionClient client, CancellationToken ct)
{
for (var i = 0; i < 40; i++)
{
var r = await client.PublishAsync(H(), default, ct);
if (r.NotificationMessage.NotificationData.Count > 0) return r;
}
throw new InvalidOperationException("no data message");
}
private static async Task<List<(uint Handle, DataValue Value)>> CollectAsync(SessionClient client, CancellationToken ct, TimeSpan window)
{
var collected = new List<(uint, DataValue)>();
var until = DateTime.UtcNow + window;
while (DateTime.UtcNow < until)
{
var r = await client.PublishAsync(H(), default, ct);
collected.AddRange(DataChanges(r.NotificationMessage));
}
return collected;
}
}
@@ -0,0 +1,250 @@
using Opc.Ua;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
/// <summary>
/// Gitea #717 spec §4.6 (the T0-adopted mechanism) at unit level: the ordinary-tag marker and its validation, the
/// Standby rewrite, the (SubscriptionId, SequenceNumber) decision record and its bounds, and the redundancy option
/// rules of spec §3.
/// </summary>
public sealed class WarmStandbyUnitTests
{
private static DataValue Good(double v, int secondsAgo = 0) =>
new(new Variant(v), StatusCodes.Good, (DateTimeUtc)DateTime.UtcNow.AddSeconds(-secondsAgo), DateTimeUtc.Now);
private static MonitoredItemNotification Marked(uint handle, DataValue value)
{
var n = new MonitoredItemNotification { ClientHandle = handle, Value = value };
n.Message = new OrdinaryTagMarker(handle, value);
return n;
}
private static NotificationMessage Message(uint seq, params MonitoredItemNotification[] items) => new()
{
SequenceNumber = seq,
PublishTime = DateTimeUtc.Now,
NotificationData = new List<ExtensionObject>
{
new(new DataChangeNotification { MonitoredItems = items.ToList() }),
},
};
private static List<MonitoredItemNotification> Items(NotificationMessage m)
{
m.NotificationData[0].TryGetValue<DataChangeNotification>(out var d).ShouldBeTrue();
return Enumerable.Range(0, d!.MonitoredItems.Count).Select(i => d.MonitoredItems[i]).ToList();
}
// ------------------------------------------------------------------ marker + rewrite
[Fact]
public void A_marked_ordinary_tag_notification_is_rewritten_and_the_input_is_not_mutated()
{
var value = Good(5);
var tag = Marked(1, value);
var other = new MonitoredItemNotification { ClientHandle = 2, Value = Good(6) };
var input = Message(7, tag, other);
var projected = StandbyNotificationRewriter.Project(input, out var rewritten);
rewritten.ShouldBe(1);
ReferenceEquals(projected, input).ShouldBeFalse();
var items = Items(projected);
items[0].ClientHandle.ShouldBe(1u);
items[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService);
items[0].Value.WrappedValue.IsNull.ShouldBeTrue();
items[0].Value.ServerTimestamp.ShouldBe(value.ServerTimestamp);
ReferenceEquals(items[1], other).ShouldBeTrue("a non-ordinary notification is passed through untouched");
projected.SequenceNumber.ShouldBe(7u);
// The SDK-owned input (the per-delivery clone) is untouched.
Items(input)[0].Value.StatusCode.ShouldBe(StatusCodes.Good);
ReferenceEquals(Items(input)[0], tag).ShouldBeTrue();
}
[Fact]
public void A_stale_marker_from_a_recycled_payload_is_ignored()
{
// 2.0.1 pools MonitoredItemNotification payloads and Initialize() resets only ClientHandle and Value, so a
// recycled object can carry a marker made for another notification. Handle or value mismatch ⇒ not ordinary.
var otherHandle = new MonitoredItemNotification { ClientHandle = 3, Value = Good(5) };
otherHandle.Message = new OrdinaryTagMarker(1, otherHandle.Value);
var otherValue = new MonitoredItemNotification { ClientHandle = 1, Value = Good(6) };
otherValue.Message = new OrdinaryTagMarker(1, Good(5, secondsAgo: 10));
OrdinaryTagMarker.IsOrdinaryTag(otherHandle).ShouldBeFalse();
OrdinaryTagMarker.IsOrdinaryTag(otherValue).ShouldBeFalse();
var input = Message(1, otherHandle, otherValue);
ReferenceEquals(StandbyNotificationRewriter.Project(input, out var rewritten), input).ShouldBeTrue();
rewritten.ShouldBe(0);
}
[Fact]
public void Bad_and_unmarked_notifications_are_never_rewritten()
{
var bad = Marked(1, new DataValue(Variant.Null, StatusCodes.BadNoCommunication, DateTimeUtc.Now, DateTimeUtc.Now));
var unmarked = new MonitoredItemNotification { ClientHandle = 2, Value = Good(1) };
var input = Message(1, bad, unmarked);
ReferenceEquals(StandbyNotificationRewriter.Project(input, out _), input).ShouldBeTrue();
}
[Fact]
public void Uncertain_ordinary_tag_values_are_projected_too()
{
var uncertain = Marked(1, new DataValue(new Variant(1.0), StatusCodes.UncertainLastUsableValue, DateTimeUtc.Now, DateTimeUtc.Now));
Items(StandbyNotificationRewriter.Project(Message(1, uncertain), out _))[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService);
}
// ------------------------------------------------------------------ SDK-field contract (regression pin)
/// <summary>
/// The mechanism stands on two SDK 2.0.1 behaviours of <see cref="MonitoredItemNotification.Message"/>: the SDK's
/// per-delivery <c>CoreUtils.Clone</c> COPIES it, and the binary encoder NEVER writes it. An SDK upgrade that
/// changes either breaks D4 silently (no marker ⇒ no rewrite) or leaks a server object onto the wire — this
/// fails loudly instead.
/// </summary>
[Fact]
public void Sdk_contract_the_marker_survives_the_delivery_clone_and_is_never_encoded()
{
var value = Good(5);
var marked = Message(9, Marked(1, value));
var clone = CoreUtils.Clone(marked);
ReferenceEquals(clone, marked).ShouldBeFalse();
var cloned = Items(clone!)[0];
ReferenceEquals(cloned, Items(marked)[0]).ShouldBeFalse("the clone is deep");
cloned.Message.ShouldBeOfType<OrdinaryTagMarker>("the SDK clone must copy MonitoredItemNotification.Message");
OrdinaryTagMarker.IsOrdinaryTag(cloned).ShouldBeTrue("the copied marker still validates against the clone");
var context = ServiceMessageContext.Create(DefaultTelemetry.Create(_ => { }));
var unmarked = Message(9, new MonitoredItemNotification { ClientHandle = 1, Value = value });
unmarked.PublishTime = marked.PublishTime;
var withMarker = BinaryEncoder.EncodeMessage(marked, context);
var without = BinaryEncoder.EncodeMessage(unmarked, context);
withMarker.ShouldBe(without, "the helper field must never reach the wire");
var decoded = BinaryDecoder.DecodeMessage<NotificationMessage>(withMarker, context);
Items(decoded!)[0].Message.ShouldBeNull();
}
// ------------------------------------------------------------------ decision record
[Fact]
public void The_first_delivery_records_the_decision_and_every_later_delivery_reuses_it()
{
var record = new FirstDeliveryRecord(standby: false);
record.Decide(1, 10).ShouldBeFalse();
record.SetStandby(true);
record.Decide(1, 10).ShouldBeFalse("a Republish of a message first delivered Active stays Active (D4)");
record.Decide(1, 11).ShouldBeTrue("a first delivery after the flip is Standby");
record.SetStandby(false);
record.Decide(1, 11).ShouldBeTrue("a Republish of a message first delivered Standby stays projected");
record.Decide(2, 11).ShouldBeFalse("keyed by subscription too");
}
[Fact]
public void The_record_is_pruned_by_available_sequence_numbers_and_bounded()
{
var record = new FirstDeliveryRecord(standby: true);
for (uint s = 1; s <= 5; s++) record.Decide(1, s);
record.Prune(1, [3, 4], justDelivered: 5);
record.Count.ShouldBe(3);
for (uint s = 100; s < 100 + FirstDeliveryRecord.MaxPerSubscription + 50; s++) record.Decide(1, s);
record.Count.ShouldBe(FirstDeliveryRecord.MaxPerSubscription);
record.Decide(2, 1);
record.SweepDead(new HashSet<uint> { 2 });
record.SubscriptionCount.ShouldBe(1);
}
[Fact]
public void Dead_subscriptions_are_evicted_least_recently_used_beyond_the_bound()
{
var record = new FirstDeliveryRecord(standby: false);
for (uint sub = 1; sub <= FirstDeliveryRecord.MaxSubscriptions; sub++) record.Decide(sub, 1);
record.Decide(1, 2); // touch subscription 1 so 2 is the least recently used
record.Decide(FirstDeliveryRecord.MaxSubscriptions + 1, 1);
record.SubscriptionCount.ShouldBe(FirstDeliveryRecord.MaxSubscriptions);
}
/// <summary>
/// A role flip and a first delivery race (spec §4.6 item 2): the flip waits for a decision in progress, so the
/// decision is ordered strictly before or after it — never torn. Driven by a deterministic barrier, not sleeps.
/// </summary>
[Fact]
public async Task A_flip_waits_for_a_decision_in_progress_and_orders_strictly_against_it()
{
for (var round = 0; round < 50; round++)
{
var record = new FirstDeliveryRecord(standby: false);
using var inside = new ManualResetEventSlim();
using var release = new ManualResetEventSlim();
record.DecisionBarrier = () => { inside.Set(); release.Wait(TimeSpan.FromSeconds(10)); };
var ct = TestContext.Current.CancellationToken;
var decision = Task.Run(() => record.Decide(1, (uint)round + 1), ct);
inside.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue();
var flip = Task.Run(() => record.SetStandby(true), ct);
(await Task.WhenAny(flip, Task.Delay(100, ct))).ShouldNotBe(flip, "the flip must wait for the decision in progress");
release.Set();
(await decision).ShouldBeFalse("the decision read the state before the flip");
await flip;
record.DecisionBarrier = null;
record.Decide(1, (uint)round + 1).ShouldBeFalse("and recorded it");
record.Decide(1, (uint)round + 1000).ShouldBeTrue("a later first delivery sees the flip");
}
}
// ------------------------------------------------------------------ WarmStandbyState
[Fact]
public void The_Off_state_never_projects_suppresses_or_refuses()
{
var off = new WarmStandbyState(false);
off.DataProjected = true;
off.AlarmsSuppressed = true;
off.DataProjected.ShouldBeFalse();
off.AlarmsSuppressed.ShouldBeFalse();
off.RefusesMutations.ShouldBeFalse();
off.Delivery.ShouldBeNull();
var warm = new WarmStandbyState(true);
warm.DataProjected.ShouldBeTrue("a Warm node boots Standby");
warm.AlarmsSuppressed.ShouldBeTrue();
warm.RefusesMutations.ShouldBeTrue();
warm.Delivery!.Standby.ShouldBeTrue();
}
// ------------------------------------------------------------------ OpcUa:Redundancy rules (spec §3)
[Theory]
[InlineData("Unset", new string[0], true)]
[InlineData("", new string[0], true)]
[InlineData("none", new string[0], true)]
[InlineData("None", new[] { "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:peer" }, true)]
[InlineData("WARM", new[] { "urn:peer", "urn:peer2" }, true)]
[InlineData("Warm", new string[0], false)]
[InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)]
[InlineData("Warm", new[] { "urn:self" }, false)]
[InlineData("Hot", new[] { "urn:peer" }, false)]
[InlineData("2", new[] { "urn:peer" }, false)]
[InlineData("Transparent", new[] { "urn:peer" }, false)]
public void Redundancy_option_rules(string mode, string[] peers, bool valid)
{
var options = new OpcUaRedundancyOptions { Mode = mode };
options.Check("urn:self", peers).Count.ShouldBe(valid ? 0 : 1, string.Join("; ", options.Check("urn:self", peers)));
}
[Fact]
public void Redundancy_defaults_are_Unset_with_promote_on()
{
var options = new OpcUaApplicationHostOptions();
options.Redundancy.ParsedMode.ShouldBe(OpcUaRedundancyMode.Unset);
options.Redundancy.PromoteWhilePrimaryUnreachable.ShouldBeTrue();
options.Redundancy.ToSettings().ShouldBe(WarmRedundancySettings.Unset);
}
}
@@ -0,0 +1,220 @@
using System.Collections.Concurrent;
using System.Text;
using Akka.Actor;
using Akka.TestKit;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
/// <summary>
/// Gitea #717 spec §4.1–§4.3 and §4.7 in <see cref="OpcUaPublishActor"/>: the Warm verdict from membership + the
/// accepted revisioned snapshot, the bands, and the transition step order — with a recording projection and
/// ServiceLevel publisher writing into ONE ordered log, so the invariant "an Active-band level is never visible over
/// projected data" is checked on the actual sequence.
/// </summary>
public sealed class OpcUaPublishActorWarmTests : RuntimeActorTestBase
{
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
private readonly ConcurrentQueue<string> _log = new();
private long _seq;
private NodeId Self
{
get
{
var cluster = Akka.Cluster.Cluster.Get(Sys);
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
}
}
private IActorRef Start(WarmRedundancySettings redundancy, LoggingPublisher? publisher = null) =>
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher ?? new LoggingPublisher(_log), localNode: Self, dbLess: true,
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: redundancy,
standbyProjection: new RecordingProjection(_log)));
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
new(new[]
{
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Self, DateTime.UtcNow),
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Other, DateTime.UtcNow),
},
CorrelationId.NewId(),
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
private string[] Steps => _log.ToArray();
private void AwaitLogEndsWith(params string[] tail) =>
AwaitAssert(() => Steps.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", Steps)), PresenceBudget,
cancellationToken: TestContext.Current.CancellationToken);
private static readonly string[] Promotion = ["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL:250"];
private static readonly string[] Demotion = ["SL:90", "delivery:True", "alarms:True", "proj:True", "queue:True"];
[Fact]
public void Promotion_and_demotion_run_in_the_spec_order()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
Steps.ShouldBe(Promotion, "boot is Standby; the first accepted snapshot promotes and nothing else ran");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith(Demotion);
AssertNoActiveLevelOverProjectedData();
}
/// <summary>Spec §4.7 invariant over the whole sequence: every published level ≥ 100 comes after the projection was
/// last opened, never while it is closed.</summary>
private void AssertNoActiveLevelOverProjectedData()
{
var projected = true; // a Warm node boots projected
foreach (var step in Steps)
{
if (step == "proj:True") projected = true;
else if (step == "proj:False") projected = false;
else if (step.StartsWith("SL:", StringComparison.Ordinal) && byte.Parse(step[3..]) >= 100)
projected.ShouldBeFalse($"an Active-band level was visible over projected data: {string.Join(" ", Steps)}");
}
}
[Fact]
public void An_unrevisioned_snapshot_never_makes_a_Warm_node_Active()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self, revisioned: false));
AwaitLogEndsWith("SL:90");
Steps.ShouldBe(new[] { "SL:90" });
}
/// <summary>Spec §4.2: after a newer snapshot names this node Primary, an OLDER incarnation's late publish naming the
/// peer is dropped — the node stays Active. Then a genuinely newer one demotes it (positive control).</summary>
[Fact]
public void An_older_revision_after_a_newer_one_is_dropped()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
actor.Tell(Snapshot(Self, upNumber: 5));
AwaitLogEndsWith(Promotion);
actor.Tell(Snapshot(Other, upNumber: 4)); // a departed older singleton's late heartbeat
actor.Tell(Snapshot(Self, upNumber: 5)); // a fence: processed after the stale one
AwaitAssert(() => Steps.Count(s => s == "SL:250").ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
Steps.ShouldNotContain("delivery:True", "the stale snapshot must not demote");
actor.Tell(Snapshot(Other, upNumber: 6));
AwaitLogEndsWith(Demotion);
}
/// <summary>Spec §4.7: if the Active level cannot be published, the promotion stops before it and retries; the
/// Standby level stays visible meanwhile.</summary>
[Fact]
public void A_failed_Active_level_publish_halts_before_it_and_retries()
{
var publisher = new LoggingPublisher(_log) { FailNext = 1, FailLevel = 250 };
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
actor.Tell(Snapshot(Self));
AwaitAssert(() => Steps.ShouldContain("SL:250"), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
var log = Steps;
log.Take(6).ShouldBe(["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL-FAILED:250"]);
log.Last().ShouldBe("SL:250", "retried without a new snapshot");
}
/// <summary>A demotion continues when the Standby level publish fails: failing safe degrades the data.</summary>
[Fact]
public void A_failed_Standby_level_publish_still_degrades_the_data()
{
var publisher = new LoggingPublisher(_log);
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
publisher.FailNext = 1;
publisher.FailLevel = 90;
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL-FAILED:90", "delivery:True", "alarms:True", "proj:True", "queue:True");
}
[Fact]
public void A_burst_of_alternating_verdicts_converges_on_the_last()
{
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
for (var i = 0; i < 20; i++) actor.Tell(Snapshot(i % 2 == 0 ? Self : Other));
actor.Tell(Snapshot(Self));
AwaitLogEndsWith(Promotion);
AssertNoActiveLevelOverProjectedData();
}
/// <summary>The default-off pin at the actor: Unset never touches the projection and keeps today's levels.</summary>
[Fact]
public void Unset_never_drives_the_projection_and_keeps_todays_levels()
{
var actor = Start(WarmRedundancySettings.Unset);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("SL:240");
Steps.ShouldBe(new[] { "SL:250", "SL:240" });
}
[Fact]
public void A_deployed_cluster_mode_that_disagrees_with_the_local_mode_is_logged_and_counted()
{
var artifact = Encoding.UTF8.GetBytes(
"{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":\"None\"}],\"Nodes\":[]}");
DeploymentArtifact.ReadClusterRedundancyMode(artifact, "x:1").ShouldBe("None");
DeploymentArtifact.ReadClusterRedundancyMode(
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":1}]}"), "x:1").ShouldBe("Warm");
DeploymentArtifact.ReadClusterRedundancyMode(
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"a\",\"RedundancyMode\":\"None\"},{\"ClusterId\":\"b\",\"RedundancyMode\":\"Hot\"}],"
+ "\"Nodes\":[{\"NodeId\":\"x:1\",\"ClusterId\":\"b\"}]}"), "x:1").ShouldBe("Hot");
using var listener = new System.Diagnostics.Metrics.MeterListener();
long mismatches = 0;
listener.InstrumentPublished = (instrument, l) =>
{
if (ReferenceEquals(instrument, ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.RedundancyModeMismatch))
l.EnableMeasurementEvents(instrument);
};
listener.SetMeasurementEventCallback<long>((_, value, _, _) => Interlocked.Add(ref mismatches, value));
listener.Start();
var warmActor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
localNode: NodeId.Parse("x:1"), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm)));
EventFilter.Error(contains: "RedundancyMode for this node's cluster is None").ExpectOne(() =>
warmActor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), Artifact: artifact)),
TestContext.Current.CancellationToken);
Interlocked.Read(ref mismatches).ShouldBeGreaterThanOrEqualTo(1);
}
}
/// <summary>Writes each published level into the shared log; can be told to fail the next N publishes of a level.</summary>
internal sealed class LoggingPublisher(ConcurrentQueue<string> log) : IServiceLevelPublisher
{
public int FailNext { get; set; }
public byte FailLevel { get; set; }
public void Publish(byte serviceLevel)
{
if (FailNext > 0 && serviceLevel == FailLevel)
{
FailNext--;
log.Enqueue($"SL-FAILED:{serviceLevel}");
throw new InvalidOperationException("injected ServiceLevel publish failure");
}
log.Enqueue($"SL:{serviceLevel}");
}
}
@@ -0,0 +1,91 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
/// <summary>Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7).</summary>
public sealed class WarmRoleAuthorityTests
{
private static readonly NodeId Self = NodeId.Parse("a:1");
private static readonly NodeId Peer = NodeId.Parse("b:1");
private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true,
NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) =>
new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote);
[Fact]
public void Active_iff_membership_and_the_accepted_snapshot_both_name_self()
{
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf)
{
var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self));
verdict.Role.ShouldBe(WarmRole.Standby);
verdict.Disagreement.ShouldBeTrue();
}
[Fact]
public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership()
{
var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null));
verdict.Role.ShouldBe(WarmRole.Standby);
verdict.Reason.ShouldBe("boot");
}
[Fact]
public void Not_up_or_endpoint_failed_is_Standby()
{
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby);
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby);
}
/// <summary>Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes.</summary>
[Fact]
public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option()
{
var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true);
WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby,
"a reachable Primary is never overridden");
WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role
.ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby");
}
[Fact]
public void A_late_bind_replays_the_retained_state_but_not_the_actions()
{
var deferred = new DeferredStandbyProjection();
deferred.SetDataProjection(false);
deferred.SetStandbyDelivery(false);
deferred.SetAlarmSuppression(false);
deferred.QueueCurrentToLiveItems(false).ShouldBe(0);
deferred.RefreshAllEventItems().ShouldBe(0);
var inner = new RecordingProjection();
deferred.SetInner(inner);
inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" });
deferred.QueueCurrentToLiveItems(true);
inner.Log.Last().ShouldBe("queue:True");
}
}
/// <summary>Records every <see cref="IStandbyProjection"/> step into a shared, ordered log.</summary>
internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue<string>? log = null) : IStandbyProjection
{
public System.Collections.Concurrent.ConcurrentQueue<string> Log { get; } = log ?? new();
public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}");
public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}");
public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}");
public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; }
public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; }
}