feat(717): Warm non-transparent redundancy, default-off (T1-T10)
OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start). - T1 options + validator + effective-URI check + boot banner; Warm needs the driver role. - T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None and the SDK's orphan i=11314 deleted; Unset untouched. - T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active. - T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable. - T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec 4.7 order (Active level last and retried; Standby level first, data degraded regardless). - T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState. - T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's cached copy is never mutated. - T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters. - T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion refresh via ISubscription.ConditionRefresh2Async on every live event item. - T10 deploy-time RedundancyMode cross-check + six instruments. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -62,4 +62,44 @@ public static class ServiceLevelCalculator
|
||||
|
||||
return (byte)Math.Clamp(basis + (h.IsDriverPrimary ? 10 : 0), 0, 255);
|
||||
}
|
||||
|
||||
/// <summary>Warm Standby, healthy (Gitea #717 spec §4.3): Degraded band.</summary>
|
||||
public const byte WarmStandbyHealthy = 90;
|
||||
|
||||
/// <summary>Warm Standby, stale or DB unreachable (spec §4.3): Degraded band.</summary>
|
||||
public const byte WarmStandbyDegraded = 50;
|
||||
|
||||
/// <summary>Warm Active, critically degraded (spec §4.3): the lowest level a data-serving Active node publishes.</summary>
|
||||
public const byte WarmActiveCritical = 100;
|
||||
|
||||
/// <summary>Warm Active, stale config with the DB reachable (spec §4.3).</summary>
|
||||
public const byte WarmActiveStale = 200;
|
||||
|
||||
/// <summary>Warm Active, healthy (spec §4.3): 240 + the 10-point Primary bonus.</summary>
|
||||
public const byte WarmActiveHealthy = 250;
|
||||
|
||||
/// <summary>
|
||||
/// The Warm ServiceLevel bands (Gitea #717 spec §4.3). <see cref="Compute"/> is unchanged and still serves
|
||||
/// <c>Unset</c> and <c>None</c>; this is used only under <c>OpcUa:Redundancy:Mode=Warm</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Invariant: every Standby level (≤ <see cref="WarmStandbyHealthy"/>) is below every level a data-serving
|
||||
/// Active node publishes (≥ <see cref="WarmActiveCritical"/>), so a client that picks the highest
|
||||
/// ServiceLevel never prefers a Standby. A member that is not Up/Joining, or whose own endpoint failed, is 0
|
||||
/// whatever its role (unchanged from <see cref="Compute"/>, lmxopcua#762).
|
||||
/// </remarks>
|
||||
/// <param name="h">The node's health facts; <see cref="NodeHealthInputs.IsDriverPrimary"/> is ignored — the
|
||||
/// Warm verdict <paramref name="isActive"/> replaces it.</param>
|
||||
/// <param name="isActive">The Warm role verdict (spec §4.1).</param>
|
||||
/// <returns>The ServiceLevel byte.</returns>
|
||||
public static byte ComputeWarm(NodeHealthInputs h, bool isActive)
|
||||
{
|
||||
if (h.MemberState is not (MemberStatus.Up or MemberStatus.Joining)) return 0;
|
||||
if (!h.EndpointListening) return 0;
|
||||
|
||||
var healthy = h.DbReachable && h.OpcUaProbeOk && !h.Stale;
|
||||
if (!isActive) return healthy ? WarmStandbyHealthy : WarmStandbyDegraded;
|
||||
if (healthy) return WarmActiveHealthy;
|
||||
return h.DbReachable && h.Stale ? WarmActiveStale : WarmActiveCritical;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,74 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
/// Broadcast whenever the cluster's redundancy topology changes (node up/down, role-leader
|
||||
/// change, partition heal). Subscribers compute their local OPC UA ServiceLevel from this.
|
||||
/// </summary>
|
||||
/// <param name="Nodes">One entry per cluster member.</param>
|
||||
/// <param name="CorrelationId">A random id per publish (diagnostics only; it carries no order).</param>
|
||||
/// <param name="Revision">The publisher's ordered revision (Gitea #717, spec §4.2). <c>null</c> from a pre-#717
|
||||
/// publisher during a rolling upgrade: such a snapshot still feeds the existing gates, but can never make a Warm
|
||||
/// node Active.</param>
|
||||
public sealed record RedundancyStateChanged(
|
||||
IReadOnlyList<NodeRedundancyState> Nodes,
|
||||
CorrelationId CorrelationId);
|
||||
CorrelationId CorrelationId,
|
||||
SnapshotRevision? Revision = null);
|
||||
|
||||
/// <summary>
|
||||
/// The order of <see cref="RedundancyStateChanged"/> snapshots (Gitea #717, spec §4.2), compared
|
||||
/// lexicographically. Within one cluster incarnation the singleton always lives on the oldest member, and every
|
||||
/// surviving member has a larger up-number than a departed oldest one, so <see cref="PublisherUpNumber"/> is
|
||||
/// monotonic across handovers; <see cref="IncarnationTick"/> orders a singleton restarted in the same process;
|
||||
/// <see cref="Sequence"/> orders the publishes of one incarnation.
|
||||
/// </summary>
|
||||
/// <param name="PublisherUpNumber">The <c>Member.UpNumber</c> of the node hosting the singleton.</param>
|
||||
/// <param name="IncarnationTick"><c>Environment.TickCount64</c> at the singleton's <c>PreStart</c>.</param>
|
||||
/// <param name="Sequence">Incremented on every publish, heartbeats included.</param>
|
||||
public sealed record SnapshotRevision(long PublisherUpNumber, long IncarnationTick, long Sequence)
|
||||
: IComparable<SnapshotRevision>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public int CompareTo(SnapshotRevision? other)
|
||||
{
|
||||
if (other is null) return 1;
|
||||
var c = PublisherUpNumber.CompareTo(other.PublisherUpNumber);
|
||||
if (c != 0) return c;
|
||||
c = IncarnationTick.CompareTo(other.IncarnationTick);
|
||||
return c != 0 ? c : Sequence.CompareTo(other.Sequence);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString() => $"{PublisherUpNumber}.{IncarnationTick}.{Sequence}";
|
||||
}
|
||||
|
||||
/// <summary>Why a snapshot was not accepted by a <see cref="SnapshotRevisionGate"/>.</summary>
|
||||
public enum SnapshotRejection
|
||||
{
|
||||
/// <summary>Accepted.</summary>
|
||||
None,
|
||||
|
||||
/// <summary>Its revision is not greater than the last accepted one (a late publish of an older incarnation).</summary>
|
||||
OlderRevision,
|
||||
|
||||
/// <summary>It carries no revision, while revisioned snapshots are being ordered.</summary>
|
||||
Unrevisioned,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The subscriber-side order rule of spec §4.2: accept a snapshot only if its revision is greater than the last
|
||||
/// accepted one. Not thread-safe; owned by one actor.
|
||||
/// </summary>
|
||||
public sealed class SnapshotRevisionGate
|
||||
{
|
||||
/// <summary>The last accepted revision; <c>null</c> until a revisioned snapshot was accepted.</summary>
|
||||
public SnapshotRevision? LastAccepted { get; private set; }
|
||||
|
||||
/// <summary>Decide one snapshot. An unrevisioned snapshot is reported <see cref="SnapshotRejection.Unrevisioned"/>
|
||||
/// and never advances the order (the caller decides whether the existing gates still use it).</summary>
|
||||
/// <param name="msg">The snapshot.</param>
|
||||
/// <returns><see cref="SnapshotRejection.None"/> when accepted.</returns>
|
||||
public SnapshotRejection Offer(RedundancyStateChanged msg)
|
||||
{
|
||||
if (msg.Revision is not { } revision) return SnapshotRejection.Unrevisioned;
|
||||
if (LastAccepted is { } last && revision.CompareTo(last) <= 0) return SnapshotRejection.OlderRevision;
|
||||
LastAccepted = revision;
|
||||
return SnapshotRejection.None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,6 +190,44 @@ public static class OtOpcUaTelemetry
|
||||
Meter.CreateCounter<long>("otopcua.opcua.condition_events.superseded", unit: "{delivery}",
|
||||
description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state.");
|
||||
|
||||
// ---------------- Warm redundancy (Gitea #717, spec §7) ----------------
|
||||
|
||||
private static int _warmRole = -1;
|
||||
|
||||
/// <summary>Records this process's Warm role for <see cref="WarmRole"/>: 1 Active, 0 Standby, −1 not Warm.</summary>
|
||||
/// <param name="value">The role value.</param>
|
||||
public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value);
|
||||
|
||||
/// <summary>The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm.</summary>
|
||||
public static readonly ObservableGauge<int> WarmRole =
|
||||
Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}",
|
||||
description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm.");
|
||||
|
||||
/// <summary>Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed).</summary>
|
||||
public static readonly Counter<long> WarmTransition =
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.warm_transition", unit: "{transition}",
|
||||
description: "Warm role transitions (direction=promote|demote, reason).");
|
||||
|
||||
/// <summary>Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned).</summary>
|
||||
public static readonly Counter<long> SnapshotRejected =
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}",
|
||||
description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned).");
|
||||
|
||||
/// <summary>The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.</summary>
|
||||
public static readonly Counter<long> RedundancyModeMismatch =
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.mode_mismatch", unit: "{apply}",
|
||||
description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.");
|
||||
|
||||
/// <summary>Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4).</summary>
|
||||
public static readonly Counter<long> StandbyProjectedNotifications =
|
||||
Meter.CreateCounter<long>("otopcua.opcua.standby.projected_notifications", unit: "{notification}",
|
||||
description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery.");
|
||||
|
||||
/// <summary>New alarm condition events not delivered on a Warm Standby (still stamped).</summary>
|
||||
public static readonly Counter<long> StandbySuppressedAlarmEvents =
|
||||
Meter.CreateCounter<long>("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}",
|
||||
description: "New alarm condition events suppressed on a Warm standby.");
|
||||
|
||||
// ---------------- Convenience helpers ----------------
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// The data-plane side of a Warm role transition (Gitea #717, spec §4.7). <c>OpcUaPublishActor</c> owns the
|
||||
/// verdict and drives these steps in a fixed order; the node manager implements them. Every member is
|
||||
/// non-blocking and idempotent. Under <see cref="OpcUaRedundancyMode.Unset"/> and
|
||||
/// <see cref="OpcUaRedundancyMode.None"/> nothing calls it.
|
||||
/// </summary>
|
||||
public interface IStandbyProjection
|
||||
{
|
||||
/// <summary>Opens (<c>false</c>) or closes (<c>true</c>) the per-tag read projection: while projected, every
|
||||
/// ordinary-tag <c>Value</c> read and sample answers <c>BadOutOfService</c>/null, and writes and Part 9 condition
|
||||
/// methods are refused <c>BadInvalidState</c>. The stored value keeps updating from acquisition.</summary>
|
||||
/// <param name="projected">Whether ordinary-tag values are projected.</param>
|
||||
void SetDataProjection(bool projected);
|
||||
|
||||
/// <summary>Switches the first-delivery decision (spec §4.6, D4): a Publish response delivered for the first time
|
||||
/// while <paramref name="standby"/> is recorded as Standby and its ordinary-tag notifications are rewritten.</summary>
|
||||
/// <param name="standby">Whether first deliveries are decided Standby.</param>
|
||||
void SetStandbyDelivery(bool standby);
|
||||
|
||||
/// <summary>Suppresses (<c>true</c>) or restores delivery of NEW alarm condition events (spec §4.5). Condition
|
||||
/// state keeps being stamped, so a later refresh carries current, ackable state.</summary>
|
||||
/// <param name="suppressed">Whether new alarm events are suppressed.</param>
|
||||
void SetAlarmSuppression(bool suppressed);
|
||||
|
||||
/// <summary>Queues each live, non-stood-down ordinary-tag data item one value, ignoring its filters: the current
|
||||
/// stored value (promotion) or <c>BadOutOfService</c>/null (demotion).</summary>
|
||||
/// <param name="projected">Whether to queue the projected value instead of the stored one.</param>
|
||||
/// <returns>The number of items queued to.</returns>
|
||||
int QueueCurrentToLiveItems(bool projected);
|
||||
|
||||
/// <summary>Runs a ConditionRefresh on every live event item (Server-object, folder and direct), through the
|
||||
/// existing refresh paths and their ACL decisions (spec §4.5 promotion refresh).</summary>
|
||||
/// <returns>The number of event items refreshed.</returns>
|
||||
int RefreshAllEventItems();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Late-binding <see cref="IStandbyProjection"/> (the <see cref="DeferredServiceLevelPublisher"/> pattern): the
|
||||
/// actor gets it at construction, the node manager only exists after SDK start. The latest requested
|
||||
/// projection / delivery / suppression state is retained and replayed on bind (the #719 lesson); the two
|
||||
/// queue/refresh actions are not replayed — before bind there are no items to act on.
|
||||
/// </summary>
|
||||
public sealed class DeferredStandbyProjection : IStandbyProjection
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private IStandbyProjection? _inner;
|
||||
private bool? _projected;
|
||||
private bool? _standby;
|
||||
private bool? _suppressed;
|
||||
|
||||
/// <summary>Bind (and replay the retained state), or unbind with <c>null</c>.</summary>
|
||||
/// <param name="inner">The live implementation, or null to unbind.</param>
|
||||
public void SetInner(IStandbyProjection? inner)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_inner = inner;
|
||||
if (inner is null) return;
|
||||
if (_projected is { } p) inner.SetDataProjection(p);
|
||||
if (_standby is { } s) inner.SetStandbyDelivery(s);
|
||||
if (_suppressed is { } a) inner.SetAlarmSuppression(a);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetDataProjection(bool projected)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_projected = projected;
|
||||
_inner?.SetDataProjection(projected);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetStandbyDelivery(bool standby)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_standby = standby;
|
||||
_inner?.SetStandbyDelivery(standby);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetAlarmSuppression(bool suppressed)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_suppressed = suppressed;
|
||||
_inner?.SetAlarmSuppression(suppressed);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public int QueueCurrentToLiveItems(bool projected)
|
||||
{
|
||||
IStandbyProjection? inner;
|
||||
lock (_gate) inner = _inner;
|
||||
return inner?.QueueCurrentToLiveItems(projected) ?? 0;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public int RefreshAllEventItems()
|
||||
{
|
||||
IStandbyProjection? inner;
|
||||
lock (_gate) inner = _inner;
|
||||
return inner?.RefreshAllEventItems() ?? 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// The parsed <c>OpcUa:Redundancy:Mode</c> (Gitea #717). Startup-static: changing it requires a restart.
|
||||
/// </summary>
|
||||
public enum OpcUaRedundancyMode
|
||||
{
|
||||
/// <summary>Not opted in (the default): byte-for-byte the pre-#717 server. Nothing in the redundancy metadata,
|
||||
/// ServiceLevel bands, projection or alarm delivery changes.</summary>
|
||||
Unset = 0,
|
||||
|
||||
/// <summary>Standalone: advertises <c>RedundancySupport=None</c> with no <c>ServerUriArray</c>; never projects.</summary>
|
||||
None = 1,
|
||||
|
||||
/// <summary>A 2-node pair with a degraded standby (spec §4): non-transparent Warm metadata, Standby bands below
|
||||
/// every Active level, ordinary-tag values projected <c>BadOutOfService</c> on the Standby, new alarm events
|
||||
/// suppressed there, and writes / Part 9 methods refused <c>BadInvalidState</c> there.</summary>
|
||||
Warm = 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The runtime half of <c>OpcUa:Redundancy</c> (Gitea #717): what the redundancy actors need. The Host builds it
|
||||
/// from the validated <c>OpcUaApplicationHostOptions</c> and registers it as a singleton; a node that registers
|
||||
/// none (a harness, an admin-only node) is <see cref="Unset"/>.
|
||||
/// </summary>
|
||||
/// <param name="Mode">The redundancy mode.</param>
|
||||
/// <param name="PromoteWhilePrimaryUnreachable">Warm only: let the survivor go Active while the Primary is
|
||||
/// unreachable but not yet downed (spec §4.1, owner answer Q2). Default on.</param>
|
||||
public sealed record WarmRedundancySettings(OpcUaRedundancyMode Mode, bool PromoteWhilePrimaryUnreachable = true)
|
||||
{
|
||||
/// <summary>The not-opted-in default.</summary>
|
||||
public static WarmRedundancySettings Unset { get; } = new(OpcUaRedundancyMode.Unset);
|
||||
|
||||
/// <summary>True when this node runs the Warm role contract.</summary>
|
||||
public bool IsWarm => Mode == OpcUaRedundancyMode.Warm;
|
||||
}
|
||||
@@ -37,6 +37,11 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
private readonly Action<object>? _broadcastOverride;
|
||||
private readonly TimeSpan _heartbeatInterval;
|
||||
private bool _dirty;
|
||||
/// <summary>Gitea #717 spec §4.2: <c>Environment.TickCount64</c> at <see cref="PreStart"/>, ordering a singleton
|
||||
/// restarted in the same process.</summary>
|
||||
private long _incarnationTick;
|
||||
/// <summary>Gitea #717 spec §4.2: incremented on every publish, heartbeats included.</summary>
|
||||
private long _sequence;
|
||||
|
||||
/// <summary>Gets the timer scheduler for this actor.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
@@ -78,6 +83,7 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
_incarnationTick = NextIncarnationTick();
|
||||
_cluster.Subscribe(
|
||||
Self,
|
||||
ClusterEvent.InitialStateAsEvents,
|
||||
@@ -107,12 +113,63 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
_dirty = false;
|
||||
|
||||
var snapshot = BuildSnapshot();
|
||||
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId());
|
||||
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId(), NextRevision());
|
||||
if (_broadcastOverride is not null) _broadcastOverride(msg);
|
||||
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(Topic, msg));
|
||||
_log.Debug("Published RedundancyStateChanged with {Count} nodes", snapshot.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ordered revision of the next publish (Gitea #717 spec §4.2): the up-number of the member hosting this
|
||||
/// singleton, this incarnation's start tick, and a per-publish sequence. Subscribers drop anything not newer,
|
||||
/// so a late heartbeat from an old incarnation can no longer overwrite the new incarnation's snapshot.
|
||||
/// </summary>
|
||||
private SnapshotRevision NextRevision()
|
||||
{
|
||||
long upNumber;
|
||||
try
|
||||
{
|
||||
upNumber = ReadUpNumber(_cluster.SelfMember)
|
||||
?? throw new InvalidOperationException("Akka.Cluster Member.UpNumber is not readable on this Akka version");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Not expected on a running singleton; an unordered 0 can never outrank a real publisher.
|
||||
_log.Debug(ex, "RedundancyState: SelfMember up-number unavailable; publishing revision up-number 0");
|
||||
upNumber = 0;
|
||||
}
|
||||
|
||||
return new SnapshotRevision(upNumber, _incarnationTick, ++_sequence);
|
||||
}
|
||||
|
||||
private static long _lastIncarnationTick;
|
||||
|
||||
/// <summary><c>Environment.TickCount64</c>, made strictly increasing within the process so a singleton restarted
|
||||
/// inside one tick still outranks its predecessor (whose sequence may be ahead of the new one's).</summary>
|
||||
/// <returns>The incarnation tick.</returns>
|
||||
public static long NextIncarnationTick()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var last = Interlocked.Read(ref _lastIncarnationTick);
|
||||
var next = Math.Max(Environment.TickCount64, last + 1);
|
||||
if (Interlocked.CompareExchange(ref _lastIncarnationTick, next, last) == last) return next;
|
||||
}
|
||||
}
|
||||
|
||||
private static readonly System.Reflection.PropertyInfo? UpNumberProperty = typeof(Member).GetProperty(
|
||||
"UpNumber", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic);
|
||||
|
||||
/// <summary>
|
||||
/// A member's up-number — the order in which members became Up, the basis of <c>Member.IsOlderThan</c> and of
|
||||
/// singleton placement. Akka.NET 1.5 keeps the property <c>internal</c>, so it is read by reflection; the
|
||||
/// <c>SnapshotRevisionTests</c> pin that it resolves, so an Akka upgrade that renames it fails loudly.
|
||||
/// </summary>
|
||||
/// <param name="member">The member.</param>
|
||||
/// <returns>The up-number, or null if the property is unavailable.</returns>
|
||||
public static int? ReadUpNumber(Member member) =>
|
||||
UpNumberProperty?.GetValue(member) is int value ? value : null;
|
||||
|
||||
/// <summary>The cluster role that marks a node as carrying the driver runtime.</summary>
|
||||
public const string DriverRole = RoleParser.Driver;
|
||||
|
||||
|
||||
+5
@@ -60,5 +60,10 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase<
|
||||
|| lockout.MaxFailedAttemptsPerClient >= lockout.MaxFailedAttempts,
|
||||
"OpcUa:CredentialLockout:MaxFailedAttemptsPerClient must be 0 (= 2 x MaxFailedAttempts) or at least "
|
||||
+ "OpcUa:CredentialLockout:MaxFailedAttempts");
|
||||
|
||||
// Gitea #717 spec §3: Hot/unknown modes fail start; Warm needs distinct peers that do not include this server's
|
||||
// own ApplicationUri; None needs no peers. The effective URI is re-checked after an ApplicationConfigPath load.
|
||||
foreach (var failure in (o.Redundancy ?? new OpcUaRedundancyOptions()).Check(o.ApplicationUri, o.PeerApplicationUris))
|
||||
builder.RequireThat(false, failure);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// The one Gitea #717 startup rule the options validator cannot see: <c>OpcUa:Redundancy:Mode=Warm</c> needs the
|
||||
/// <c>driver</c> role (the validator is registered only on driver nodes, so on any other node it would never run).
|
||||
/// </summary>
|
||||
public static class OpcUaRedundancyStartupGuard
|
||||
{
|
||||
/// <summary>Fails host start when a node without the driver role is configured Warm.</summary>
|
||||
/// <param name="configuredMode">The raw <c>OpcUa:Redundancy:Mode</c> value.</param>
|
||||
/// <param name="hasDriver">Whether this node carries the driver role.</param>
|
||||
/// <exception cref="InvalidOperationException">Warm without the driver role.</exception>
|
||||
public static void RequireDriverRoleForWarm(string? configuredMode, bool hasDriver)
|
||||
{
|
||||
if (hasDriver) return;
|
||||
var options = new OpcUaRedundancyOptions { Mode = configuredMode ?? string.Empty };
|
||||
if (options.TryParseMode(out var mode) && mode == OpcUaRedundancyMode.Warm)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"OpcUa:Redundancy:Mode is Warm on a node without the driver role; Warm redundancy needs the driver role "
|
||||
+ "(the OPC UA endpoint is served by driver nodes only). Remove the setting or add the driver role.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
private readonly OpcUaApplicationHostOptions _options;
|
||||
private readonly DeferredAddressSpaceSink _deferredSink;
|
||||
private readonly DeferredServiceLevelPublisher _deferredServiceLevel;
|
||||
private readonly DeferredStandbyProjection? _deferredStandby;
|
||||
private readonly IOpcUaUserAuthenticator _userAuthenticator;
|
||||
private readonly IHistorianDataSource _historianDataSource;
|
||||
private readonly IDataPlaneAuthoritySource _dataPlaneAuthority;
|
||||
@@ -127,6 +128,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
/// <param name="writeBounds">The node's configured write bounds (<c>DriverWrites:Timeout</c>); the inbound
|
||||
/// write gateway's Ask is derived from them (lmxopcua#777). Optional: null ⇒
|
||||
/// <see cref="DriverWriteBounds.Default"/>.</param>
|
||||
/// <param name="deferredStandby">The late-bound Warm standby projection (Gitea #717) <c>OpcUaPublishActor</c> drives;
|
||||
/// bound here once the SDK has started. Optional: null (and a non-Warm server) binds nothing.</param>
|
||||
public OtOpcUaServerHostedService(
|
||||
IOptions<OpcUaApplicationHostOptions> options,
|
||||
DeferredAddressSpaceSink deferredSink,
|
||||
@@ -147,8 +150,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
OpcUaEndpointState? endpointState = null,
|
||||
EffectiveRoles? roles = null,
|
||||
NativeAlarmAckPreflightView? ackPreflightView = null,
|
||||
DriverWriteBounds? writeBounds = null)
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
DeferredStandbyProjection? deferredStandby = null)
|
||||
{
|
||||
_deferredStandby = deferredStandby;
|
||||
ArgumentNullException.ThrowIfNull(aclOptions);
|
||||
ArgumentNullException.ThrowIfNull(time);
|
||||
|
||||
@@ -402,6 +407,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
serverInternal,
|
||||
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
|
||||
}
|
||||
|
||||
// Gitea #717: the Warm standby projection. Bound right after the ServiceLevel so a transition the actor
|
||||
// already decided replays onto the node manager; null (every non-Warm server) leaves the adapter unbound.
|
||||
if (_server.CreateStandbyProjection() is { } standbyProjection) _deferredStandby?.SetInner(standbyProjection);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -568,6 +577,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// half-disposed NodeManager.
|
||||
_deferredSink.SetSink(null);
|
||||
_deferredServiceLevel.SetInner(null);
|
||||
_deferredStandby?.SetInner(null);
|
||||
// Restore the Null write gateway so a late client write doesn't Ask a stopping DriverHostActor.
|
||||
_server?.SetNodeWriteGateway(null);
|
||||
// Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor.
|
||||
|
||||
@@ -206,6 +206,10 @@ ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.OpcUaUserCertificatePaths.Configure(
|
||||
// lmxopcua#765: refresh/staleness of the cached API-key → Galaxy ack identity set (driver nodes consume it).
|
||||
builder.Services.AddGalaxyAckIdentityCache(builder.Configuration);
|
||||
|
||||
// Gitea #717 spec §3: Warm redundancy is a property of the OPC UA endpoint, which only a driver node serves. A Warm
|
||||
// mode on a node without the driver role is a configuration error, not something to ignore silently.
|
||||
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(builder.Configuration["OpcUa:Redundancy:Mode"], hasDriver);
|
||||
|
||||
if (hasDriver)
|
||||
{
|
||||
builder.Services.AddOtOpcUaRuntime();
|
||||
@@ -501,6 +505,11 @@ if (hasDriver)
|
||||
builder.Services.AddValidatedOptions<OpcUaApplicationHostOptions, OpcUaApplicationHostOptionsValidator>(
|
||||
builder.Configuration, "OpcUa");
|
||||
|
||||
// Gitea #717: the runtime half of OpcUa:Redundancy for the redundancy actors, and the late-bound standby
|
||||
// projection the OPC UA hosted service binds once the SDK has started (same pattern as the ServiceLevel publisher).
|
||||
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<OpcUaApplicationHostOptions>>().Value.Redundancy.ToSettings());
|
||||
builder.Services.AddSingleton<DeferredStandbyProjection>();
|
||||
|
||||
// Lazy ActorSystem accessor so OtOpcUaServerHostedService can resolve the DistributedPubSub
|
||||
// mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
|
||||
// DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
|
||||
|
||||
@@ -6,8 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// Preserves the application's established equipment-write conversions across SDK 2.0's
|
||||
/// new pre-handler type validation. The owning callback remains the authorization and shape gate.
|
||||
/// </summary>
|
||||
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock) : BaseDataVariableState(parent)
|
||||
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock, WarmStandbyState? standby = null)
|
||||
: BaseDataVariableState(parent)
|
||||
{
|
||||
private readonly WarmStandbyState _standby = standby ?? WarmStandbyState.Off;
|
||||
|
||||
// Every field below is accessed under ownerLock. Tokens identify client attempts; the mutation
|
||||
// revision detects a driver publication that re-enters the callback before its optimistic commit.
|
||||
private long _nextWriteToken;
|
||||
@@ -82,6 +85,16 @@ internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock
|
||||
QualifiedName dataEncoding, ref Variant value, ref DateTimeUtc sourceTimestamp)
|
||||
{
|
||||
// Subscription samplers read the node directly, bypassing the manager's service lock.
|
||||
lock (ownerLock) return base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
|
||||
ServiceResult result;
|
||||
lock (ownerLock) result = base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
|
||||
|
||||
// Gitea #717 spec §4.4 (T0b): on a Warm Standby every value path — Read (IndexRange and DataEncoding included),
|
||||
// the initial sample, the change fan-out, ResendData, AggregateFilter input — answers BadOutOfService/null. The
|
||||
// stored Value is untouched and keeps updating from acquisition. The base read runs first, so an IndexRange or
|
||||
// encoding error is still reported as such. Off (Unset/None/Active): returned unchanged.
|
||||
if (ServiceResult.IsBad(result) || !_standby.DataProjected) return result;
|
||||
value = Variant.Null;
|
||||
sourceTimestamp = DateTimeUtc.MinValue;
|
||||
return StatusCodes.BadOutOfService;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using Opc.Ua;
|
||||
using Opc.Ua.Configuration;
|
||||
using Opc.Ua.Identity;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
|
||||
@@ -174,6 +175,83 @@ public sealed class OpcUaApplicationHostOptions
|
||||
/// Replaces the SDK's own lockout, which also refused Anonymous; see <see cref="FailedCredentialLockout"/>.
|
||||
/// </summary>
|
||||
public OpcUaCredentialLockoutOptions CredentialLockout { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Warm non-transparent redundancy (<c>OpcUa:Redundancy:*</c>, Gitea #717). The default <c>Mode=Unset</c> is
|
||||
/// byte-for-byte today's server. Startup-static.
|
||||
/// </summary>
|
||||
public OpcUaRedundancyOptions Redundancy { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>The <c>OpcUa:Redundancy</c> section (Gitea #717, spec §6).</summary>
|
||||
public sealed class OpcUaRedundancyOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>Unset</c> (default) / <c>None</c> / <c>Warm</c>, case-insensitive. <c>Hot</c> is not advertisable
|
||||
/// (spec §3) and, like any other value, fails host start. Bound as a string so a bad value is a readable
|
||||
/// validation failure rather than a binder exception.
|
||||
/// </summary>
|
||||
public string Mode { get; set; } = nameof(OpcUaRedundancyMode.Unset);
|
||||
|
||||
/// <summary>
|
||||
/// Warm only: let the survivor go Active while the Primary is unreachable but not yet downed (spec §4.1, owner
|
||||
/// answer Q2). Default <c>true</c>.
|
||||
/// </summary>
|
||||
public bool PromoteWhilePrimaryUnreachable { get; set; } = true;
|
||||
|
||||
/// <summary>Parses <see cref="Mode"/>.</summary>
|
||||
/// <param name="mode">The parsed mode.</param>
|
||||
/// <returns><c>false</c> for <c>Hot</c>, numerics or any unknown value.</returns>
|
||||
public bool TryParseMode(out OpcUaRedundancyMode mode)
|
||||
{
|
||||
var text = (Mode ?? string.Empty).Trim();
|
||||
if (text.Length == 0) text = nameof(OpcUaRedundancyMode.Unset);
|
||||
if (!text.All(char.IsLetter)) { mode = default; return false; }
|
||||
return Enum.TryParse(text, ignoreCase: true, out mode) && Enum.IsDefined(mode);
|
||||
}
|
||||
|
||||
/// <summary>The parsed mode; throws on an invalid one (the validator rejects it first).</summary>
|
||||
public OpcUaRedundancyMode ParsedMode => TryParseMode(out var m)
|
||||
? m
|
||||
: throw new InvalidOperationException(
|
||||
$"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable).");
|
||||
|
||||
/// <summary>The runtime settings the redundancy actors read.</summary>
|
||||
/// <returns>The settings.</returns>
|
||||
public WarmRedundancySettings ToSettings() => new(ParsedMode, PromoteWhilePrimaryUnreachable);
|
||||
|
||||
/// <summary>
|
||||
/// The rules of spec §3 that need only configuration: <c>Warm</c> needs at least one peer, all URIs distinct
|
||||
/// and the self URI not listed as a peer; <c>None</c> needs no peers. Returns the failure messages (empty when
|
||||
/// valid). The effective self URI is checked again after an <c>ApplicationConfigPath</c> load.
|
||||
/// </summary>
|
||||
/// <param name="selfUri">The self URI to check against.</param>
|
||||
/// <param name="peers">The configured peer URIs.</param>
|
||||
/// <returns>The failures.</returns>
|
||||
public IReadOnlyList<string> Check(string? selfUri, IEnumerable<string>? peers)
|
||||
{
|
||||
var failures = new List<string>();
|
||||
if (!TryParseMode(out var mode))
|
||||
{
|
||||
failures.Add($"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable: continuous acquisition alone does not meet Part 4 Hot)");
|
||||
return failures;
|
||||
}
|
||||
|
||||
var list = (peers ?? Array.Empty<string>()).Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToList();
|
||||
if (mode == OpcUaRedundancyMode.None && list.Count > 0)
|
||||
failures.Add("OpcUa:PeerApplicationUris must be empty when OpcUa:Redundancy:Mode is None (a standalone server has no peers)");
|
||||
if (mode == OpcUaRedundancyMode.Warm)
|
||||
{
|
||||
if (list.Count == 0)
|
||||
failures.Add("OpcUa:Redundancy:Mode Warm requires at least one OpcUa:PeerApplicationUris entry");
|
||||
if (list.Distinct(StringComparer.Ordinal).Count() != list.Count)
|
||||
failures.Add("OpcUa:PeerApplicationUris must not contain duplicates under OpcUa:Redundancy:Mode Warm");
|
||||
if (!string.IsNullOrWhiteSpace(selfUri) && list.Contains(selfUri.Trim(), StringComparer.Ordinal))
|
||||
failures.Add($"OpcUa:PeerApplicationUris must not list this server's own ApplicationUri '{selfUri}' under OpcUa:Redundancy:Mode Warm (each pair node needs a unique ApplicationUri)");
|
||||
}
|
||||
|
||||
return failures;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -289,7 +367,8 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
|
||||
|
||||
try
|
||||
{
|
||||
_ = await BuildConfigurationAsync(cancellationToken);
|
||||
var configuration = await BuildConfigurationAsync(cancellationToken);
|
||||
ConfigureRedundancy(sdkServer, configuration.ApplicationUri ?? _options.ApplicationUri);
|
||||
await EnsureApplicationCertificateAsync(cancellationToken).ConfigureAwait(false);
|
||||
AttachUserAuthenticator(sdkServer);
|
||||
await _application.StartAsync(server, cancellationToken).ConfigureAwait(false);
|
||||
@@ -305,6 +384,31 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
|
||||
_options.PublicHostname, _options.OpcUaPort);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 (spec §3): validate the redundancy mode against the EFFECTIVE self URI — the loaded
|
||||
/// <c>ApplicationConfigPath</c> XML wins over <c>OpcUa:ApplicationUri</c> (c7787) — log the boot banner and hand
|
||||
/// the startup-static metadata to the SDK server, which installs it in <c>OnNodeManagerStartedAsync</c>.
|
||||
/// <c>Unset</c> hands nothing over, so the SDK's graph is untouched.
|
||||
/// </summary>
|
||||
/// <param name="server">The SDK server about to start.</param>
|
||||
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
|
||||
/// <exception cref="InvalidOperationException">The redundancy configuration is invalid for the effective URI.</exception>
|
||||
private void ConfigureRedundancy(OtOpcUaSdkServer server, string effectiveSelfUri)
|
||||
{
|
||||
var redundancy = _options.Redundancy ?? new OpcUaRedundancyOptions();
|
||||
var failures = redundancy.Check(effectiveSelfUri, _options.PeerApplicationUris);
|
||||
if (failures.Count > 0)
|
||||
throw new InvalidOperationException("OPC UA redundancy configuration is invalid: " + string.Join("; ", failures));
|
||||
|
||||
var mode = redundancy.ParsedMode;
|
||||
var peers = (_options.PeerApplicationUris ?? new List<string>())
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToArray();
|
||||
_logger.LogInformation(
|
||||
"OPC UA redundancy: Mode={Mode}, effective ApplicationUri={SelfUri}, peers=[{Peers}], PromoteWhilePrimaryUnreachable={Promote}",
|
||||
mode, effectiveSelfUri, string.Join(", ", peers), redundancy.PromoteWhilePrimaryUnreachable);
|
||||
server.ConfigureRedundancy(mode, effectiveSelfUri, peers);
|
||||
}
|
||||
|
||||
private void AttachUserAuthenticator(OtOpcUaSdkServer server)
|
||||
{
|
||||
foreach (var type in new[] { UserTokenType.Anonymous, UserTokenType.UserName })
|
||||
|
||||
@@ -161,6 +161,34 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
private FolderState? _root;
|
||||
private readonly ILogger<OtOpcUaNodeManager> _logger;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
|
||||
/// <param name="server">The OPC UA server instance.</param>
|
||||
/// <param name="configuration">The application configuration.</param>
|
||||
/// <param name="logger">Logger for the manager's swallowed-fault and skipped-wiring diagnostics;
|
||||
/// <c>null</c> (tests) falls back to <see cref="NullLogger{T}.Instance"/>.</param>
|
||||
/// <param name="standby">The Warm standby flags (Gitea #717); null (every non-Warm server) is
|
||||
/// <see cref="WarmStandbyState.Off"/>, which leaves every data path exactly as it was.</param>
|
||||
internal OtOpcUaNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration, ILogger<OtOpcUaNodeManager>? logger,
|
||||
WarmStandbyState? standby)
|
||||
: this(server, configuration, logger)
|
||||
{
|
||||
_standby = standby ?? WarmStandbyState.Off;
|
||||
if (_standby.WarmMode)
|
||||
{
|
||||
// Spec §4.6 item 1: ordinary-tag Value items are built by OrdinaryTagMonitoredItem, which marks each
|
||||
// notification. Warm only — Unset/None keep the SDK's own manager.
|
||||
m_monitoredItemManager.Dispose();
|
||||
m_monitoredItemManager = new OrdinaryTagItemManager(this.ToAsyncNodeManager(), server);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The Warm standby flags (Gitea #717). <see cref="WarmStandbyState.Off"/> unless the server is Warm.</summary>
|
||||
private readonly WarmStandbyState _standby = WarmStandbyState.Off;
|
||||
|
||||
/// <summary>Test observation: the Warm standby flags.</summary>
|
||||
internal WarmStandbyState Standby => _standby;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
|
||||
/// <param name="server">The OPC UA server instance.</param>
|
||||
/// <param name="configuration">The application configuration.</param>
|
||||
@@ -1434,6 +1462,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
// the InstanceStateSnapshot captures the just-projected values.
|
||||
alarm.ClearChangeMasks(SystemContext, includeChildren: true);
|
||||
|
||||
// Gitea #717 spec §4.5: on a Warm Standby only the STAMP half above runs — the condition node is current and a
|
||||
// later refresh carries a valid, ackable EventId — and the DELIVER half below does not. Suppressing here, before
|
||||
// the snapshot and every SDK field projection, covers the Server-object, folder and direct paths, ACL on or
|
||||
// off, by construction; overflow, filtered retain and the #838 deferral queue never see the event. The
|
||||
// decision is made at report time: an event reported while Active is delivered even if that ends after
|
||||
// demotion.
|
||||
if (_standby.AlarmsSuppressed)
|
||||
{
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbySuppressedAlarmEvents.Add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// InstanceStateSnapshot is the IFilterTarget — a frozen copy of the condition's fields at fire
|
||||
@@ -2991,6 +3031,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
var gate = EvaluateAlarmCommandGate(identity, condition, operation);
|
||||
if (gate is not null) return gate;
|
||||
|
||||
// Gitea #717 owner answer Q4: a Warm Standby refuses every Part 9 condition method before it changes any state
|
||||
// (nothing routed, nothing applied, no event). Unset/None keep today's acceptance.
|
||||
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal($"call {operation}");
|
||||
|
||||
var cmd = new AlarmCommand(
|
||||
AlarmId: condition.NodeId.IdentifierAsString,
|
||||
Operation: operation,
|
||||
@@ -3032,6 +3076,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
var gate = EvaluateAlarmCommandGate(identity, condition, "Acknowledge");
|
||||
if (gate is not null) return gate;
|
||||
|
||||
// Gitea #717 owner answer Q4: refused on a Warm Standby, ahead of (and whatever) the Primary gate below says.
|
||||
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("acknowledge the alarm");
|
||||
|
||||
// PRIMARY GATE, after authorization (a caller who may not ack learns nothing about the
|
||||
// node's role). Only the Primary routes a native ack upstream (DriverHostActor applies the same verdict
|
||||
// to the route), so on any other node the ack is refused outright rather than applied locally and then
|
||||
@@ -3215,6 +3262,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
?? EvaluateEquipmentWriteGate(identity, gatewayWired);
|
||||
if (gate is not null) return gate;
|
||||
|
||||
// Gitea #717 owner answer Q1: a Warm Standby refuses the write synchronously, AFTER authorization (a caller who
|
||||
// may not write learns nothing about the node's role) and before anything reaches the driver. Nothing is
|
||||
// applied, so nothing is reverted. Unset/None never set the flag and keep the #5 optimistic path.
|
||||
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("write the value");
|
||||
|
||||
// Item A (synchronous structural fail-fast): reject a structurally-invalid write INLINE — return Bad
|
||||
// synchronously so the SDK never applies it (no optimistic-Good-then-revert round-trip + no needless
|
||||
// device dispatch). Runs AFTER the authz gate (so we never leak structure detail to an unauthorised
|
||||
@@ -4120,7 +4172,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
// The SDK exposes the flags separately (no CurrentReadWrite composite): ReadWrite is
|
||||
// CurrentRead | CurrentWrite, with the HistoryRead bit OR-ed in when historized.
|
||||
var access = ComposeAccessLevel(writable, historized, HistoryWritable(_historianDataUpdater));
|
||||
var variable = new EquipmentVariableState(parent, Lock)
|
||||
var variable = new EquipmentVariableState(parent, Lock, _standby)
|
||||
{
|
||||
NodeId = new NodeId(variableNodeId, ns),
|
||||
BrowseName = new QualifiedName(variableNodeId, ns),
|
||||
@@ -5725,6 +5777,88 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
lock (Lock) return MonitoredNodes.ContainsKey(nodeId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.7 promotion step 2 / demotion step 5: queues one value to every live, non-stood-down
|
||||
/// ordinary-tag <c>Value</c> item, ignoring its filters (the queue pattern of <see cref="RevalidateMonitoredItems"/>
|
||||
/// — never detach or <c>Disabled</c>, which drops the queue). Promotion queues the current stored value read
|
||||
/// through the item's own IndexRange/DataEncoding; demotion queues <c>BadOutOfService</c>/null.
|
||||
/// </summary>
|
||||
/// <param name="projected">Queue the projected value (demotion) instead of the stored one (promotion).</param>
|
||||
/// <returns>The number of items queued to.</returns>
|
||||
internal int QueueCurrentToLiveItems(bool projected)
|
||||
{
|
||||
var queued = 0;
|
||||
foreach (var (item, tracked) in _liveItems)
|
||||
{
|
||||
if (tracked.Suppressed || tracked.Handle.Node is not EquipmentVariableState variable) continue;
|
||||
if (item is not IDataChangeMonitoredItem2 { AttributeId: Attributes.Value } data) continue;
|
||||
var now = UtcNow;
|
||||
DataValue value;
|
||||
if (projected)
|
||||
{
|
||||
value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, now);
|
||||
}
|
||||
else
|
||||
{
|
||||
var stored = new DataValue();
|
||||
ServiceResult read;
|
||||
lock (Lock) read = variable.ReadAttribute(SystemContext, Attributes.Value, data.IndexRange, data.DataEncoding, ref stored);
|
||||
value = ServiceResult.IsBad(read)
|
||||
? new DataValue(Variant.Null, read.StatusCode, DateTimeUtc.MinValue, now)
|
||||
: new DataValue(stored.WrappedValue, stored.StatusCode, stored.SourceTimestamp, now);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
item.QueueValue(value, StatusCode.IsBad(value.StatusCode) ? new ServiceResult(value.StatusCode) : null, ignoreFilters: true);
|
||||
queued++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A disabled item refuses the value; it re-samples the (projected or not) node when re-enabled.
|
||||
_logger.LogDebug(ex, "OtOpcUaNodeManager: could not queue a Warm transition value to a monitored item");
|
||||
}
|
||||
}
|
||||
|
||||
return queued;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.5 promotion refresh: a ConditionRefresh2 on every live event item — Server-object items
|
||||
/// and the direct/folder items of this manager's notifiers — through the SDK's own subscription path, so each
|
||||
/// client sees RefreshStart, the retained conditions (through this manager's ACL decision and the #854/#855
|
||||
/// cut), then RefreshEnd. Must not be called under <c>Lock</c>.
|
||||
/// </summary>
|
||||
/// <returns>The number of event items refreshed.</returns>
|
||||
internal int RefreshAllEventItems()
|
||||
{
|
||||
var items = new List<IEventMonitoredItem>(_allEventsItems.Keys);
|
||||
foreach (var (item, tracked) in _eventItems)
|
||||
if (!tracked.Suppressed) items.Add(item);
|
||||
if (items.Count == 0) return 0;
|
||||
|
||||
var subscriptions = new Dictionary<uint, ISubscription>();
|
||||
foreach (var subscription in Server.SubscriptionManager.GetSubscriptions()) subscriptions[subscription.Id] = subscription;
|
||||
|
||||
var refreshed = 0;
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (!subscriptions.TryGetValue(item.SubscriptionId, out var subscription)) continue;
|
||||
try
|
||||
{
|
||||
subscription.ConditionRefresh2Async(item.Id).AsTask().GetAwaiter().GetResult();
|
||||
refreshed++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A subscription deleted mid-walk, or an item whose mode refuses a refresh: that client re-refreshes itself.
|
||||
_logger.LogDebug(ex, "OtOpcUaNodeManager: Warm promotion refresh of event item {ItemId} failed", item.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return refreshed;
|
||||
}
|
||||
|
||||
/// <summary>Test observability: how many data-change items the revalidation map tracks.</summary>
|
||||
internal int LiveItemCount => _liveItems.Count;
|
||||
|
||||
|
||||
@@ -315,6 +315,160 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
/// </summary>
|
||||
public const byte StartupServiceLevel = 0;
|
||||
|
||||
private OpcUaRedundancyMode _redundancyMode = OpcUaRedundancyMode.Unset;
|
||||
private string _selfUri = string.Empty;
|
||||
private string[] _peerUris = [];
|
||||
private WarmStandbyState _standby = WarmStandbyState.Off;
|
||||
|
||||
/// <summary>The redundancy mode configured before start (Gitea #717).</summary>
|
||||
public OpcUaRedundancyMode RedundancyMode => _redundancyMode;
|
||||
|
||||
/// <summary>The Warm standby flags shared with the node manager.</summary>
|
||||
internal WarmStandbyState Standby => _standby;
|
||||
|
||||
/// <summary>The initial <c>Server.ServiceLevel</c> under <c>Warm</c> (spec §4.3: the NoData band, replacing the
|
||||
/// SDK's 255 and this server's #885 value 0 until the first computed level).</summary>
|
||||
public const byte WarmBootServiceLevel = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Sets the startup-static redundancy metadata (Gitea #717 spec §3) before <c>StartAsync</c>.
|
||||
/// <see cref="OpcUaRedundancyMode.Unset"/> (the default when this is never called) leaves the SDK graph and
|
||||
/// every data path exactly as they are today.
|
||||
/// </summary>
|
||||
/// <param name="mode">The validated mode.</param>
|
||||
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
|
||||
/// <param name="peerUris">The peer URIs, in configured order.</param>
|
||||
public void ConfigureRedundancy(OpcUaRedundancyMode mode, string effectiveSelfUri, IReadOnlyList<string> peerUris)
|
||||
{
|
||||
if (_otOpcUaNodeManager is not null)
|
||||
throw new InvalidOperationException("Redundancy must be configured before the server starts.");
|
||||
_redundancyMode = mode;
|
||||
_selfUri = effectiveSelfUri;
|
||||
_peerUris = peerUris.ToArray();
|
||||
_standby = new WarmStandbyState(mode == OpcUaRedundancyMode.Warm);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The data-plane transition steps a Warm <c>OpcUaPublishActor</c> drives (spec §4.7), or null when the server
|
||||
/// is not Warm or has not started.
|
||||
/// </summary>
|
||||
/// <returns>The projection, or null.</returns>
|
||||
public IStandbyProjection? CreateStandbyProjection() =>
|
||||
_standby.WarmMode && _otOpcUaNodeManager is { } nm ? new SdkStandbyProjection(nm, _standby) : null;
|
||||
|
||||
/// <summary>Test seam: ids of the live subscriptions, for the decision record's dead-subscription sweep.</summary>
|
||||
private IReadOnlySet<uint> LiveSubscriptionIds()
|
||||
{
|
||||
var ids = new HashSet<uint>();
|
||||
try
|
||||
{
|
||||
if (CurrentInstance?.SubscriptionManager is { } manager)
|
||||
foreach (var subscription in manager.GetSubscriptions()) ids.Add(subscription.Id);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Shutdown race: an empty set only drops records, which re-decide (fail-safe) at their next delivery.
|
||||
}
|
||||
|
||||
return ids;
|
||||
}
|
||||
|
||||
private long _publishesSinceSweep;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask<PublishResponse> PublishAsync(SecureChannelContext secureChannelContext,
|
||||
RequestHeader? requestHeader, ArrayOf<SubscriptionAcknowledgement> subscriptionAcknowledgements,
|
||||
RequestLifetime requestLifetime)
|
||||
{
|
||||
var response = await base.PublishAsync(secureChannelContext, requestHeader, subscriptionAcknowledgements, requestLifetime)
|
||||
.ConfigureAwait(false);
|
||||
if (_standby.Delivery is not { } record || response?.NotificationMessage is not { } message) return response!;
|
||||
|
||||
if (message.NotificationData.Count > 0)
|
||||
{
|
||||
response.NotificationMessage = Decide(record, response.SubscriptionId, message);
|
||||
var available = new List<uint>(response.AvailableSequenceNumbers.Count);
|
||||
for (var i = 0; i < response.AvailableSequenceNumbers.Count; i++) available.Add(response.AvailableSequenceNumbers[i]);
|
||||
record.Prune(response.SubscriptionId, available, message.SequenceNumber);
|
||||
}
|
||||
|
||||
if (Interlocked.Increment(ref _publishesSinceSweep) % 1024 == 0) record.SweepDead(LiveSubscriptionIds());
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask<RepublishResponse> RepublishAsync(SecureChannelContext secureChannelContext,
|
||||
RequestHeader? requestHeader, uint subscriptionId, uint retransmitSequenceNumber, RequestLifetime requestLifetime)
|
||||
{
|
||||
var response = await base.RepublishAsync(secureChannelContext, requestHeader, subscriptionId, retransmitSequenceNumber,
|
||||
requestLifetime).ConfigureAwait(false);
|
||||
if (_standby.Delivery is { } record && response?.NotificationMessage is { NotificationData.Count: > 0 } message)
|
||||
response.NotificationMessage = Decide(record, subscriptionId, message);
|
||||
return response!;
|
||||
}
|
||||
|
||||
/// <summary>The recorded (or, first time, current) decision for one delivery, applied to the SDK's fresh clone.</summary>
|
||||
private NotificationMessage Decide(FirstDeliveryRecord record, uint subscriptionId, NotificationMessage message)
|
||||
{
|
||||
if (!record.Decide(subscriptionId, message.SequenceNumber)) return message;
|
||||
var projected = StandbyNotificationRewriter.Project(message, out var rewritten);
|
||||
if (rewritten > 0)
|
||||
{
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbyProjectedNotifications.Add(rewritten);
|
||||
}
|
||||
|
||||
return projected;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Installs the standard redundancy metadata (Gitea #717 spec §3, T0a): under <c>Warm</c>
|
||||
/// <c>Server.ServerRedundancy</c> becomes a <c>NonTransparentRedundancyType</c> whose read-only
|
||||
/// <c>ServerUriArray</c> (<c>i=11314</c>) is <c>[self, peers…]</c>, <c>RedundancySupport</c> is <c>Warm</c> and
|
||||
/// ServiceLevel boots at <see cref="WarmBootServiceLevel"/>; under <c>None</c> <c>RedundancySupport</c> is
|
||||
/// <c>None</c> and the SDK's orphan <c>i=11314</c> is deleted, so it is absent by NodeId as well as by browse.
|
||||
/// <c>Unset</c> touches nothing.
|
||||
/// </summary>
|
||||
private async ValueTask InstallRedundancyMetadataAsync(IServerInternal server, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_redundancyMode == OpcUaRedundancyMode.Unset) return;
|
||||
var serverObject = server.ServerObject
|
||||
?? throw new InvalidOperationException("The Server object is unavailable during server startup.");
|
||||
var context = server.DefaultSystemContext;
|
||||
var redundancy = serverObject.ServerRedundancy
|
||||
?? throw new InvalidOperationException("Server.ServerRedundancy is unavailable during server startup.");
|
||||
|
||||
if (_redundancyMode == OpcUaRedundancyMode.None)
|
||||
{
|
||||
redundancy.RedundancySupport!.Value = RedundancySupport.None;
|
||||
redundancy.RedundancySupport.ClearChangeMasks(context, false);
|
||||
// T0a finding: the SDK's predefined i=11314 survives as an orphan (unreferenced, but Good [] by NodeId).
|
||||
if (server.DiagnosticsNodeManager is AsyncCustomNodeManager diagnostics)
|
||||
await diagnostics.DeleteNodeAsync(context, VariableIds.Server_ServerRedundancy_ServerUriArray, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var replacer = server.DiagnosticsNodeManager as IPredefinedNodeSubtypeReplacer
|
||||
?? throw new InvalidOperationException("The diagnostics node manager cannot replace predefined node subtypes.");
|
||||
var replaced = (NonTransparentRedundancyState)await replacer.ReplacePredefinedInstanceSubtypeAsync(context, redundancy,
|
||||
context.CreateInstanceOfNonTransparentRedundancyType(),
|
||||
new Dictionary<QualifiedName, NodeId>
|
||||
{
|
||||
[new QualifiedName(BrowseNames.ServerUriArray, 0)] = VariableIds.Server_ServerRedundancy_ServerUriArray,
|
||||
},
|
||||
node => serverObject.ServerRedundancy = (ServerRedundancyState)node, cancellationToken).ConfigureAwait(false);
|
||||
replaced.RedundancySupport!.Value = RedundancySupport.Warm;
|
||||
replaced.ServerUriArray!.Value = (ArrayOf<string>)new[] { _selfUri }.Concat(_peerUris).ToArray();
|
||||
replaced.ServerUriArray.AccessLevel = replaced.ServerUriArray.UserAccessLevel = AccessLevels.CurrentRead;
|
||||
replaced.ClearChangeMasks(context, true);
|
||||
if (serverObject.ServiceLevel is { } serviceLevel)
|
||||
{
|
||||
serviceLevel.Value = WarmBootServiceLevel;
|
||||
serviceLevel.Timestamp = DateTime.UtcNow;
|
||||
serviceLevel.ClearChangeMasks(context, includeChildren: false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async ValueTask OnNodeManagerStartedAsync(
|
||||
IServerInternal server, CancellationToken cancellationToken = default)
|
||||
@@ -329,6 +483,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
serviceLevel.Timestamp = DateTime.UtcNow;
|
||||
serviceLevel.ClearChangeMasks(server.DefaultSystemContext, includeChildren: false);
|
||||
}
|
||||
await InstallRedundancyMetadataAsync(server, cancellationToken).ConfigureAwait(false);
|
||||
var auditType = server.DiagnosticsNodeManager?
|
||||
.FindPredefinedNode<BaseObjectTypeState>(ObjectTypeIds.AuditWriteUpdateEventType)
|
||||
?? throw new InvalidOperationException("AuditWriteUpdateEventType is unavailable during server startup.");
|
||||
@@ -360,7 +515,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
protected override MasterNodeManager CreateMasterNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
{
|
||||
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger);
|
||||
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger, _standby);
|
||||
return new MasterNodeManager(server, configuration, dynamicNamespaceUri: null, _otOpcUaNodeManager);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The node-manager-wide Warm standby flags (Gitea #717, spec §4.4–§4.6). One instance per server. Under
|
||||
/// <c>Unset</c>/<c>None</c> <see cref="WarmMode"/> is false and nothing ever sets a flag, so every reader sees
|
||||
/// today's behaviour. Under <c>Warm</c> the node starts Standby (all flags set) until the redundancy layer
|
||||
/// promotes it.
|
||||
/// </summary>
|
||||
internal sealed class WarmStandbyState
|
||||
{
|
||||
private volatile bool _dataProjected;
|
||||
private volatile bool _alarmsSuppressed;
|
||||
|
||||
/// <summary>Initializes the state.</summary>
|
||||
/// <param name="warmMode">Whether this server runs the Warm role contract.</param>
|
||||
public WarmStandbyState(bool warmMode)
|
||||
{
|
||||
WarmMode = warmMode;
|
||||
_dataProjected = warmMode;
|
||||
_alarmsSuppressed = warmMode;
|
||||
Delivery = warmMode ? new FirstDeliveryRecord(standby: true) : null;
|
||||
}
|
||||
|
||||
/// <summary>The not-Warm state (no projection, no suppression, no decision record).</summary>
|
||||
public static WarmStandbyState Off { get; } = new(false);
|
||||
|
||||
/// <summary>Whether this server runs the Warm role contract.</summary>
|
||||
public bool WarmMode { get; }
|
||||
|
||||
/// <summary>Ordinary-tag reads/samples project <c>BadOutOfService</c>; writes and Part 9 methods are refused.</summary>
|
||||
public bool DataProjected
|
||||
{
|
||||
get => _dataProjected;
|
||||
set => _dataProjected = WarmMode && value;
|
||||
}
|
||||
|
||||
/// <summary>New alarm condition events are not delivered (still stamped).</summary>
|
||||
public bool AlarmsSuppressed
|
||||
{
|
||||
get => _alarmsSuppressed;
|
||||
set => _alarmsSuppressed = WarmMode && value;
|
||||
}
|
||||
|
||||
/// <summary>Whether a Warm Standby must refuse a write or a Part 9 method (owner answers Q1/Q4).</summary>
|
||||
public bool RefusesMutations => WarmMode && _dataProjected;
|
||||
|
||||
/// <summary>The first-delivery decision record; null when not Warm.</summary>
|
||||
public FirstDeliveryRecord? Delivery { get; }
|
||||
|
||||
/// <summary>The refusal returned to a write or a Part 9 method on a Warm Standby.</summary>
|
||||
public static ServiceResult StandbyRefusal(string what) => new(StatusCodes.BadInvalidState,
|
||||
new LocalizedText($"This server is the Warm redundancy standby; {what} on the active server (the one with the higher ServiceLevel)."));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ordinary-tag marker (spec §4.6 item 1): set on <see cref="MonitoredItemNotification.Message"/>, a helper
|
||||
/// reference the binary encoder never writes and the SDK's per-delivery <c>CoreUtils.Clone</c> copies
|
||||
/// (pinned by <c>WarmStandbySdkFieldContractTests</c>). Valid only for the notification whose
|
||||
/// <see cref="ClientHandle"/> and <see cref="Value"/> it captured, so a stale marker left on a pooled payload
|
||||
/// (2.0.1's <c>Initialize()</c> does not clear the helper) fails <see cref="Matches"/>.
|
||||
/// </summary>
|
||||
/// <remarks>Derives from <see cref="NotificationMessage"/> only because that is the helper's declared type.</remarks>
|
||||
internal sealed class OrdinaryTagMarker : NotificationMessage
|
||||
{
|
||||
/// <summary>Initializes the marker.</summary>
|
||||
/// <param name="clientHandle">The notification's client handle when it was built.</param>
|
||||
/// <param name="value">The notification's final value when it was built.</param>
|
||||
public OrdinaryTagMarker(uint clientHandle, DataValue value)
|
||||
{
|
||||
ClientHandle = clientHandle;
|
||||
Value = value;
|
||||
}
|
||||
|
||||
/// <summary>The client handle captured at build time.</summary>
|
||||
public uint ClientHandle { get; }
|
||||
|
||||
/// <summary>The value captured at build time.</summary>
|
||||
public DataValue Value { get; }
|
||||
|
||||
/// <summary>Whether <paramref name="n"/> is the notification this marker was made for.</summary>
|
||||
/// <param name="n">A delivered notification.</param>
|
||||
/// <returns>True for a match on client handle and value.</returns>
|
||||
public bool Matches(MonitoredItemNotification n) => n.ClientHandle == ClientHandle && Equals(n.Value, Value);
|
||||
|
||||
/// <summary>Whether <paramref name="n"/> carries a valid ordinary-tag marker.</summary>
|
||||
/// <param name="n">A delivered notification.</param>
|
||||
/// <returns>True when it is an ordinary-tag notification.</returns>
|
||||
public static bool IsOrdinaryTag(MonitoredItemNotification n) => n.Message is OrdinaryTagMarker m && m.Matches(n);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The per-message first-delivery decision (spec §4.6 item 2, D4), keyed by
|
||||
/// <c>(SubscriptionId, SequenceNumber)</c> — never by object: SDK 2.0.1 hands every Publish and Republish a fresh
|
||||
/// clone of the cached message. The first delivery of a sequence number records Active or Standby; every later
|
||||
/// delivery (Republish included) applies the recorded decision. One lock orders a decision against a role flip;
|
||||
/// it is never held across an await or a projection.
|
||||
/// </summary>
|
||||
internal sealed class FirstDeliveryRecord
|
||||
{
|
||||
/// <summary>Recorded messages kept per subscription beyond what <c>AvailableSequenceNumbers</c> prunes.</summary>
|
||||
internal const int MaxPerSubscription = 1024;
|
||||
|
||||
/// <summary>Subscriptions kept before the least-recently-used one is dropped (dead subscriptions).</summary>
|
||||
internal const int MaxSubscriptions = 8192;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<uint, SubscriptionRecord> _subscriptions = new();
|
||||
private bool _standby;
|
||||
private long _touch;
|
||||
|
||||
/// <summary>Initializes the record.</summary>
|
||||
/// <param name="standby">The initial delivery state (a Warm node boots Standby).</param>
|
||||
public FirstDeliveryRecord(bool standby) => _standby = standby;
|
||||
|
||||
/// <summary>Test seam: runs inside the lock after the decision is read and before it is recorded.</summary>
|
||||
internal Action? DecisionBarrier { get; set; }
|
||||
|
||||
/// <summary>The current first-delivery state.</summary>
|
||||
public bool Standby
|
||||
{
|
||||
get { lock (_gate) return _standby; }
|
||||
}
|
||||
|
||||
/// <summary>Flips the first-delivery state (the D4 boundary). Waits for a decision in progress.</summary>
|
||||
/// <param name="standby">Whether later first deliveries are Standby.</param>
|
||||
public void SetStandby(bool standby)
|
||||
{
|
||||
lock (_gate) _standby = standby;
|
||||
}
|
||||
|
||||
/// <summary>The decision for one delivery: the recorded one, or (first delivery) the current state, recorded.</summary>
|
||||
/// <param name="subscriptionId">The subscription.</param>
|
||||
/// <param name="sequenceNumber">The message's sequence number.</param>
|
||||
/// <returns>True when the delivery must be projected (Standby).</returns>
|
||||
public bool Decide(uint subscriptionId, uint sequenceNumber)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_subscriptions.TryGetValue(subscriptionId, out var sub))
|
||||
{
|
||||
if (_subscriptions.Count >= MaxSubscriptions) EvictLeastRecentlyUsed();
|
||||
_subscriptions[subscriptionId] = sub = new SubscriptionRecord();
|
||||
}
|
||||
|
||||
sub.Touched = ++_touch;
|
||||
if (sub.Decisions.TryGetValue(sequenceNumber, out var recorded)) return recorded;
|
||||
var standby = _standby;
|
||||
DecisionBarrier?.Invoke();
|
||||
sub.Decisions[sequenceNumber] = standby;
|
||||
sub.Order.Enqueue(sequenceNumber);
|
||||
while (sub.Order.Count > MaxPerSubscription && sub.Order.TryDequeue(out var oldest)) sub.Decisions.Remove(oldest);
|
||||
return standby;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Drops the decisions of messages the subscription no longer retains (they can never be republished).</summary>
|
||||
/// <param name="subscriptionId">The subscription the Publish response belongs to.</param>
|
||||
/// <param name="available">The response's <c>AvailableSequenceNumbers</c>.</param>
|
||||
/// <param name="justDelivered">The sequence number just delivered (kept: it becomes available for Republish).</param>
|
||||
public void Prune(uint subscriptionId, IReadOnlyCollection<uint> available, uint justDelivered)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_subscriptions.TryGetValue(subscriptionId, out var sub)) return;
|
||||
var keep = new HashSet<uint>(available) { justDelivered };
|
||||
var kept = new Queue<uint>();
|
||||
while (sub.Order.TryDequeue(out var seq))
|
||||
{
|
||||
if (keep.Contains(seq)) kept.Enqueue(seq);
|
||||
else sub.Decisions.Remove(seq);
|
||||
}
|
||||
|
||||
sub.Order.Clear();
|
||||
foreach (var seq in kept) sub.Order.Enqueue(seq);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Drops every record of subscriptions not in <paramref name="live"/>.</summary>
|
||||
/// <param name="live">The ids of live subscriptions.</param>
|
||||
public void SweepDead(IReadOnlySet<uint> live)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (var id in _subscriptions.Keys.Where(id => !live.Contains(id)).ToList()) _subscriptions.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Recorded decisions (test observation).</summary>
|
||||
internal int Count
|
||||
{
|
||||
get { lock (_gate) return _subscriptions.Values.Sum(s => s.Decisions.Count); }
|
||||
}
|
||||
|
||||
/// <summary>Recorded subscriptions (test observation).</summary>
|
||||
internal int SubscriptionCount
|
||||
{
|
||||
get { lock (_gate) return _subscriptions.Count; }
|
||||
}
|
||||
|
||||
private void EvictLeastRecentlyUsed()
|
||||
{
|
||||
var victim = _subscriptions.MinBy(kv => kv.Value.Touched).Key;
|
||||
_subscriptions.Remove(victim);
|
||||
}
|
||||
|
||||
private sealed class SubscriptionRecord
|
||||
{
|
||||
public readonly Dictionary<uint, bool> Decisions = new();
|
||||
public readonly Queue<uint> Order = new();
|
||||
public long Touched;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rewrites a Standby delivery (spec §4.6): every marked ordinary-tag notification whose status is not Bad becomes
|
||||
/// <c>BadOutOfService</c>/null with its server timestamp kept. Builds new objects; the message handed in (the SDK's
|
||||
/// fresh per-delivery clone) and, a fortiori, the SDK's cached copy are never mutated.
|
||||
/// </summary>
|
||||
internal static class StandbyNotificationRewriter
|
||||
{
|
||||
/// <summary>Projects <paramref name="message"/>, or returns it unchanged when nothing needs rewriting.</summary>
|
||||
/// <param name="message">A delivered message.</param>
|
||||
/// <param name="rewritten">The number of notifications rewritten.</param>
|
||||
/// <returns>The message to deliver.</returns>
|
||||
public static NotificationMessage Project(NotificationMessage message, out int rewritten)
|
||||
{
|
||||
rewritten = 0;
|
||||
if (message is null || message.NotificationData.Count == 0) return message!;
|
||||
|
||||
var data = new List<ExtensionObject>(message.NotificationData.Count);
|
||||
var changed = false;
|
||||
for (var i = 0; i < message.NotificationData.Count; i++)
|
||||
{
|
||||
var eo = message.NotificationData[i];
|
||||
if (eo.TryGetValue<DataChangeNotification>(out var dcn) && dcn is not null)
|
||||
{
|
||||
List<MonitoredItemNotification>? items = null;
|
||||
for (var j = 0; j < dcn.MonitoredItems.Count; j++)
|
||||
{
|
||||
var n = dcn.MonitoredItems[j];
|
||||
if (OrdinaryTagMarker.IsOrdinaryTag(n) && !StatusCode.IsBad(n.Value.StatusCode))
|
||||
{
|
||||
items ??= CopyPrefix(dcn, j);
|
||||
items.Add(new MonitoredItemNotification
|
||||
{
|
||||
ClientHandle = n.ClientHandle,
|
||||
Value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, n.Value.ServerTimestamp),
|
||||
});
|
||||
rewritten++;
|
||||
}
|
||||
else
|
||||
{
|
||||
items?.Add(n);
|
||||
}
|
||||
}
|
||||
|
||||
if (items is not null)
|
||||
{
|
||||
changed = true;
|
||||
data.Add(new ExtensionObject(new DataChangeNotification { MonitoredItems = items, DiagnosticInfos = dcn.DiagnosticInfos }));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
data.Add(eo);
|
||||
}
|
||||
|
||||
if (!changed) return message;
|
||||
return new NotificationMessage { SequenceNumber = message.SequenceNumber, PublishTime = message.PublishTime, NotificationData = data };
|
||||
}
|
||||
|
||||
private static List<MonitoredItemNotification> CopyPrefix(DataChangeNotification dcn, int count)
|
||||
{
|
||||
var list = new List<MonitoredItemNotification>(dcn.MonitoredItems.Count);
|
||||
for (var k = 0; k < count; k++) list.Add(dcn.MonitoredItems[k]);
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A sampled data-change item for an ordinary-tag <c>Value</c> (spec §4.6 item 1): after the SDK's own per-value
|
||||
/// publish (so the timestamp filter has run) it marks each notification it produced with an
|
||||
/// <see cref="OrdinaryTagMarker"/>. Everything else is the SDK's <see cref="MonitoredItem"/>.
|
||||
/// </summary>
|
||||
internal sealed class OrdinaryTagMonitoredItem : MonitoredItem
|
||||
{
|
||||
/// <summary>Initializes the item exactly as <see cref="MonitoredNodeMonitoredItemManager"/> does.</summary>
|
||||
public OrdinaryTagMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager, NodeHandle handle, uint subscriptionId,
|
||||
uint id, MonitoredItemCreateRequest request, DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn,
|
||||
MonitoringFilter? filter, Opc.Ua.Range? range, double samplingInterval, uint queueSize, bool durable)
|
||||
: base(server, nodeManager, handle, subscriptionId, id, request.ItemToMonitor, diagnosticsMasks, timestampsToReturn,
|
||||
request.MonitoringMode, request.RequestedParameters.ClientHandle, filter, filter, range, samplingInterval, queueSize,
|
||||
request.RequestedParameters.DiscardOldest, 0, durable)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool Publish(OperationContext context, Queue<MonitoredItemNotification> notifications,
|
||||
Queue<DiagnosticInfo> diagnostics, uint maxNotificationsPerPublish, ILogger logger)
|
||||
{
|
||||
var local = new Queue<MonitoredItemNotification>();
|
||||
var localDiagnostics = new Queue<DiagnosticInfo>();
|
||||
var more = base.Publish(context, local, localDiagnostics, maxNotificationsPerPublish, logger);
|
||||
while (local.Count > 0)
|
||||
{
|
||||
var n = local.Dequeue();
|
||||
n.Message = new OrdinaryTagMarker(n.ClientHandle, n.Value);
|
||||
notifications.Enqueue(n);
|
||||
if (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
|
||||
}
|
||||
|
||||
while (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
|
||||
return more;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Installs <see cref="OrdinaryTagMonitoredItem"/> for ordinary-tag (<see cref="EquipmentVariableState"/>)
|
||||
/// <c>Value</c> items only. Re-implements <see cref="IMonitoredItemManager.CreateMonitoredItem"/> on the SDK's
|
||||
/// public, non-sealed manager, so <c>CustomNodeManager2</c>'s <c>is MonitoredNodeMonitoredItemManager</c> checks and
|
||||
/// its internal lifecycle interface keep working (T0c; the 2.0.1 <c>MonitoredItemFactory</c> hook exists only on
|
||||
/// <c>AsyncCustomNodeManager</c>). Installed only under Warm; <c>Unset</c>/<c>None</c> keep the SDK's manager.
|
||||
/// </summary>
|
||||
internal sealed class OrdinaryTagItemManager(IAsyncNodeManager nodeManager, IServerInternal server)
|
||||
: MonitoredNodeMonitoredItemManager(nodeManager, server), IMonitoredItemManager
|
||||
{
|
||||
/// <inheritdoc />
|
||||
ISampledDataChangeMonitoredItem IMonitoredItemManager.CreateMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager,
|
||||
ServerSystemContext context, NodeHandle handle, uint subscriptionId, double publishingInterval,
|
||||
DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn, MonitoredItemCreateRequest itemToCreate,
|
||||
Opc.Ua.Range euRange, MonitoringFilter filterToUse, double samplingInterval, uint revisedQueueSize, bool createDurable,
|
||||
MonitoredItemIdFactory monitoredItemIdFactory, Func<ISystemContext, NodeHandle, NodeState, NodeState> addNodeToComponentCache)
|
||||
{
|
||||
if (handle.Node is not EquipmentVariableState || itemToCreate.ItemToMonitor.AttributeId != Attributes.Value)
|
||||
{
|
||||
return CreateMonitoredItem(server, nodeManager, context, handle, subscriptionId, publishingInterval, diagnosticsMasks,
|
||||
timestampsToReturn, itemToCreate, euRange, filterToUse, samplingInterval, revisedQueueSize, createDurable,
|
||||
monitoredItemIdFactory, addNodeToComponentCache);
|
||||
}
|
||||
|
||||
// Mirrors MonitoredNodeMonitoredItemManager.CreateMonitoredItem in exact tagged 2.0.1, swapping only the item type.
|
||||
var cachedNode = addNodeToComponentCache(context, handle, handle.Node);
|
||||
if (!MonitoredNodes.TryGetValue(handle.Node.NodeId, out var monitoredNode))
|
||||
{
|
||||
MonitoredNodes[handle.Node.NodeId] = monitoredNode = new MonitoredNode2(nodeManager, server, cachedNode,
|
||||
nodeManager.IsMultipleEventConsumerNode(cachedNode.NodeId));
|
||||
}
|
||||
|
||||
handle.Node = monitoredNode.Node;
|
||||
handle.MonitoredNode = monitoredNode;
|
||||
uint id;
|
||||
do
|
||||
{
|
||||
id = monitoredItemIdFactory.GetNextId();
|
||||
}
|
||||
while (!MonitoredItems.TryAdd(id, null!));
|
||||
|
||||
var item = new OrdinaryTagMonitoredItem(server, nodeManager, handle, subscriptionId, id, itemToCreate, diagnosticsMasks,
|
||||
timestampsToReturn, filterToUse, euRange, samplingInterval, revisedQueueSize, createDurable);
|
||||
monitoredNode.Add((IDataChangeMonitoredItem2)item);
|
||||
MonitoredItems[id] = item;
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The <see cref="ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection"/> a Warm server hands the redundancy layer.</summary>
|
||||
internal sealed class SdkStandbyProjection(OtOpcUaNodeManager nodeManager, WarmStandbyState state)
|
||||
: ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public void SetDataProjection(bool projected) => state.DataProjected = projected;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetStandbyDelivery(bool standby) => state.Delivery?.SetStandby(standby);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetAlarmSuppression(bool suppressed) => state.AlarmsSuppressed = suppressed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int QueueCurrentToLiveItems(bool projected) => nodeManager.QueueCurrentToLiveItems(projected);
|
||||
|
||||
/// <inheritdoc />
|
||||
public int RefreshAllEventItems() => nodeManager.RefreshAllEventItems();
|
||||
}
|
||||
@@ -258,6 +258,52 @@ public static class DeploymentArtifact
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §3: the <c>ServerCluster.RedundancyMode</c> of this node's cluster as the artifact carries it
|
||||
/// (<c>None</c>/<c>Warm</c>/<c>Hot</c>), for the deploy-time cross-check against <c>OpcUa:Redundancy:Mode</c>.
|
||||
/// A single-cluster artifact answers for its only cluster; a multi-cluster one for the cluster of this node's
|
||||
/// <c>ClusterNode</c> row. Null when the artifact does not say (empty, malformed, no matching cluster).
|
||||
/// </summary>
|
||||
/// <param name="blob">The deployment artifact.</param>
|
||||
/// <param name="nodeId">This node's <c>host:port</c> id.</param>
|
||||
/// <returns>The mode name, or null.</returns>
|
||||
public static string? ReadClusterRedundancyMode(ReadOnlySpan<byte> blob, string nodeId)
|
||||
{
|
||||
if (blob.IsEmpty) return null;
|
||||
try
|
||||
{
|
||||
using var parsed = Open(blob.ToArray());
|
||||
var root = parsed.Root;
|
||||
if (!root.TryGetProperty("Clusters", out var clusters) || clusters.ValueKind != JsonValueKind.Array) return null;
|
||||
var scope = parsed.ResolveClusterScope(nodeId);
|
||||
JsonElement? match = null;
|
||||
foreach (var cluster in clusters.EnumerateArray())
|
||||
{
|
||||
if (cluster.ValueKind != JsonValueKind.Object) continue;
|
||||
if (scope.Mode == ClusterFilterMode.None) { match = cluster; break; }
|
||||
if (scope.Mode == ClusterFilterMode.ScopeTo
|
||||
&& string.Equals(cluster.TryGetProperty("ClusterId", out var cid) && cid.ValueKind == JsonValueKind.String ? cid.GetString() : null, scope.ClusterId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
match = cluster;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (match is not { } m || !m.TryGetProperty("RedundancyMode", out var mode)) return null;
|
||||
return mode.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => mode.GetString(),
|
||||
JsonValueKind.Number when mode.TryGetInt32(out var n) && Enum.IsDefined(typeof(Configuration.Enums.RedundancyMode), n)
|
||||
=> ((Configuration.Enums.RedundancyMode)n).ToString(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolve how a node should scope a deployment artifact to its own ClusterId. Single-cluster
|
||||
/// (or legacy) artifacts resolve to <see cref="ClusterFilterMode.None"/> so every existing
|
||||
|
||||
@@ -789,7 +789,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
TimeProvider? writeClock = null,
|
||||
IScriptRunner? scriptedAlarmRunner = null,
|
||||
TimeSpan? childShutdownTimeout = null) =>
|
||||
TimeSpan? childShutdownTimeout = null,
|
||||
bool enforceSnapshotOrdering = false) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -803,7 +804,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
|
||||
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout));
|
||||
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout,
|
||||
enforceSnapshotOrdering));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -938,8 +940,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
TimeProvider? writeClock = null,
|
||||
IScriptRunner? scriptedAlarmRunner = null,
|
||||
TimeSpan? childShutdownTimeout = null)
|
||||
TimeSpan? childShutdownTimeout = null,
|
||||
bool enforceSnapshotOrdering = false)
|
||||
{
|
||||
_snapshotOrdering = enforceSnapshotOrdering ? new SnapshotRevisionGate() : null;
|
||||
_childShutdownTimeout = childShutdownTimeout;
|
||||
_scriptedAlarmRunner = scriptedAlarmRunner;
|
||||
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
|
||||
@@ -2552,8 +2556,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// the cached role unchanged; when this node has a real driver peer (count > 1) that omission is the
|
||||
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
|
||||
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
|
||||
/// <summary>Gitea #717: the snapshot order gate under Warm; null (no ordering, today's behaviour) otherwise.</summary>
|
||||
private readonly SnapshotRevisionGate? _snapshotOrdering;
|
||||
|
||||
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
// Gitea #717 spec §4.2, under OpcUa:Redundancy:Mode=Warm only (Unset/None keep today's unconditional replace):
|
||||
// a snapshot older than the last accepted one — a late heartbeat of an old singleton incarnation — is dropped,
|
||||
// so it cannot overwrite the new incarnation's roles. An unrevisioned snapshot (pre-#717 publisher) still feeds
|
||||
// the gates, as before.
|
||||
if (_snapshotOrdering?.Offer(msg) == SnapshotRejection.OlderRevision)
|
||||
{
|
||||
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
|
||||
return;
|
||||
}
|
||||
|
||||
_redundancySnapshotReceived = true;
|
||||
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
|
||||
if (local is not null)
|
||||
|
||||
@@ -14,6 +14,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
|
||||
@@ -326,7 +327,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null) =>
|
||||
IOpcUaEndpointState? endpointState = null,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
@@ -341,7 +344,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
dbLess,
|
||||
dataPlaneAuthority,
|
||||
utcNow,
|
||||
endpointState)).WithDispatcher(DispatcherId);
|
||||
endpointState,
|
||||
redundancy,
|
||||
standbyProjection)).WithDispatcher(DispatcherId);
|
||||
|
||||
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
|
||||
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
|
||||
@@ -385,7 +390,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null) =>
|
||||
IOpcUaEndpointState? endpointState = null,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
@@ -400,7 +407,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
dbLess,
|
||||
dataPlaneAuthority,
|
||||
utcNow,
|
||||
endpointState));
|
||||
endpointState,
|
||||
redundancy,
|
||||
standbyProjection));
|
||||
|
||||
/// <summary>Test-only Props exposing the deferred-write bound and the plan-step seams.</summary>
|
||||
internal static Props PropsForTests(
|
||||
@@ -461,10 +470,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null)
|
||||
IOpcUaEndpointState? endpointState = null,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null)
|
||||
: this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow,
|
||||
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null,
|
||||
endpointState)
|
||||
endpointState, redundancy, standbyProjection)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -488,8 +499,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Func<DateTime>? utcNow,
|
||||
int deferredWriteCapacity,
|
||||
TestHooks? hooks,
|
||||
IOpcUaEndpointState? endpointState)
|
||||
IOpcUaEndpointState? endpointState,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null)
|
||||
{
|
||||
_redundancy = redundancy ?? WarmRedundancySettings.Unset;
|
||||
_standbyProjection = standbyProjection;
|
||||
_deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity));
|
||||
_hooks = hooks;
|
||||
_sink = sink;
|
||||
@@ -523,6 +538,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Receive<HealthTick>(_ => OnHealthTick());
|
||||
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(HandlePeerProbe);
|
||||
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
|
||||
// Gitea #717 spec §4.1 / owner answer Q2: under Warm the verdict re-derives on membership and reachability, not
|
||||
// only on snapshots. Nothing subscribes these under Unset/None.
|
||||
Receive<ClusterEvent.IMemberEvent>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.IReachabilityEvent>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.CurrentClusterState>(_ => RecomputeServiceLevel());
|
||||
Receive<WarmRetry>(_ => RecomputeServiceLevel());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -543,6 +564,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
base.PreRestart(reason, message);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
try { _cluster.Unsubscribe(Self); }
|
||||
catch (Exception) { /* the cluster is shutting down with us */ }
|
||||
}
|
||||
|
||||
base.PostStop();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
@@ -551,6 +584,20 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
|
||||
}
|
||||
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
OtOpcUaTelemetry.SetWarmRole(0);
|
||||
try
|
||||
{
|
||||
_cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents,
|
||||
typeof(ClusterEvent.IMemberEvent), typeof(ClusterEvent.IReachabilityEvent));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "OpcUaPublish: Warm redundancy could not subscribe to cluster membership; the verdict re-derives on snapshots only");
|
||||
}
|
||||
}
|
||||
|
||||
// DB-backed nodes refresh their cached probe status; DB-less nodes still need periodic
|
||||
// snapshot/probe-verdict age checks even when no new messages arrive. Legacy nodes keep
|
||||
// their role-only behavior without a timer.
|
||||
@@ -672,7 +719,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleRebuild(RebuildAddressSpace msg) => StartRebuild(msg, Sender);
|
||||
private void HandleRebuild(RebuildAddressSpace msg)
|
||||
{
|
||||
if (msg.Artifact is { Length: > 0 } artifact) CheckRedundancyMode(artifact);
|
||||
StartRebuild(msg, Sender);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The entry half of a rebuild, on the actor thread. The raw-sink fallback (no applier) and the
|
||||
@@ -1465,26 +1516,34 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleServiceLevelChanged(ServiceLevelChanged msg)
|
||||
private void HandleServiceLevelChanged(ServiceLevelChanged msg) => TryPublishServiceLevel(msg.ServiceLevel);
|
||||
|
||||
/// <summary>The dedup/publish/metric step behind <see cref="ServiceLevelChanged"/>; synchronous so a Warm
|
||||
/// transition can order it against the projection steps (spec §4.7).</summary>
|
||||
/// <param name="level">The level to publish.</param>
|
||||
/// <returns>True when the level is published (or already was).</returns>
|
||||
private bool TryPublishServiceLevel(byte level)
|
||||
{
|
||||
// Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a
|
||||
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
|
||||
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
|
||||
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
|
||||
if (_publishedAtLeastOnce && level == _lastServiceLevel) return true;
|
||||
try
|
||||
{
|
||||
_serviceLevel.Publish(msg.ServiceLevel);
|
||||
_lastServiceLevel = msg.ServiceLevel;
|
||||
_serviceLevel.Publish(level);
|
||||
_lastServiceLevel = level;
|
||||
_publishedAtLeastOnce = true;
|
||||
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
|
||||
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
|
||||
_log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel);
|
||||
new KeyValuePair<string, object?>("level", level));
|
||||
_log.Debug("OpcUaPublish: ServiceLevel={Level}", level);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failed write may be partially applied; even a previous successful value must retry.
|
||||
_publishedAtLeastOnce = false;
|
||||
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", msg.ServiceLevel);
|
||||
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", level);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1494,6 +1553,28 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// wired) or via the legacy role-only seam (back-compat / bootstrap).</summary>
|
||||
private void HandleRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
// Gitea #717 spec §4.2: under Warm a snapshot older than the last accepted one (a late heartbeat from an old
|
||||
// singleton incarnation) is dropped outright. An unrevisioned one (a pre-#717 publisher mid-upgrade) still
|
||||
// feeds the health inputs but can never make this node Active. Unset/None keep today's unconditional
|
||||
// replace below.
|
||||
switch (_revisionGate.Offer(msg))
|
||||
{
|
||||
case SnapshotRejection.OlderRevision:
|
||||
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
|
||||
_log.Debug("OpcUaPublish: dropped redundancy snapshot {Revision} (last accepted {Last})",
|
||||
msg.Revision, _revisionGate.LastAccepted);
|
||||
return;
|
||||
case SnapshotRejection.Unrevisioned:
|
||||
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "unrevisioned"));
|
||||
break;
|
||||
default:
|
||||
_acceptedSnapshot = msg;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_lastSnapshot = msg;
|
||||
RecomputeServiceLevel();
|
||||
}
|
||||
@@ -1576,6 +1657,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
private void RecomputeServiceLevel()
|
||||
{
|
||||
if (_localNode is null || _lastSnapshot is null) return;
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
RecomputeWarm();
|
||||
return;
|
||||
}
|
||||
|
||||
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
|
||||
|
||||
@@ -1635,6 +1721,197 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Warm redundancy (Gitea #717)
|
||||
|
||||
private sealed class WarmRetry { public static readonly WarmRetry Instance = new(); private WarmRetry() { } }
|
||||
|
||||
private readonly WarmRedundancySettings _redundancy;
|
||||
private readonly IStandbyProjection? _standbyProjection;
|
||||
private readonly SnapshotRevisionGate _revisionGate = new();
|
||||
/// <summary>The latest ACCEPTED revisioned snapshot — the only one the Warm verdict reads (spec §4.2).</summary>
|
||||
private RedundancyStateChanged? _acceptedSnapshot;
|
||||
/// <summary>The role the data plane is in. A Warm node manager boots Standby, so this starts Standby.</summary>
|
||||
private WarmRole _warmRole = WarmRole.Standby;
|
||||
/// <summary>A promotion whose Active ServiceLevel publish failed (spec §4.7): the data plane is Active and the
|
||||
/// level is retried; the Standby level stays visible meanwhile, never the reverse.</summary>
|
||||
private bool _promotionLevelPending;
|
||||
private DateTime? _disagreeSinceUtc;
|
||||
private bool _disagreementWarned;
|
||||
|
||||
/// <summary>How long membership and snapshot may disagree before a Warning (two heartbeats, spec §4.1).</summary>
|
||||
internal static readonly TimeSpan WarmDisagreementWarning = TimeSpan.FromSeconds(20);
|
||||
|
||||
/// <summary>The current Warm role (test observation).</summary>
|
||||
internal WarmRole CurrentWarmRole => _warmRole;
|
||||
|
||||
/// <summary>A promoted node whose Active ServiceLevel publish is being retried (test observation).</summary>
|
||||
internal bool PromotionLevelPending => _promotionLevelPending;
|
||||
|
||||
/// <summary>
|
||||
/// The Warm ServiceLevel + role path (spec §4.1, §4.3, §4.7): the health facts the non-Warm paths use, the
|
||||
/// Warm verdict, then a transition in the fixed step order when the verdict changed, else just the band.
|
||||
/// </summary>
|
||||
private void RecomputeWarm()
|
||||
{
|
||||
var self = _localNode!.Value;
|
||||
var entry = _lastSnapshot!.Nodes.FirstOrDefault(n => n.NodeId == self);
|
||||
var now = _utcNow();
|
||||
var memberState = SafeSelfStatus();
|
||||
var listening = EndpointListening();
|
||||
|
||||
bool dbReachable;
|
||||
bool stale;
|
||||
if (_dbLess || _dbHealthProbe is null || _lastDbHealth is null)
|
||||
{
|
||||
dbReachable = true;
|
||||
stale = entry is null || (now - entry.AsOfUtc) > _staleWindow;
|
||||
}
|
||||
else
|
||||
{
|
||||
dbReachable = _lastDbHealth.Reachable;
|
||||
stale = !_lastDbHealth.Reachable
|
||||
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|
||||
|| entry is null || (now - entry.AsOfUtc) > _staleWindow;
|
||||
}
|
||||
|
||||
var verdict = EvaluateWarmVerdict(self, memberState, listening);
|
||||
TrackDisagreement(verdict, now);
|
||||
|
||||
var detached = entry is null || entry.Role == RedundancyRole.Detached;
|
||||
var inputs = new NodeHealthInputs(memberState, dbReachable, OpcUaProbeOk(), stale, IsDriverPrimary: false, listening);
|
||||
var target = detached ? WarmRole.Standby : verdict.Role;
|
||||
var level = detached ? (byte)0 : ServiceLevelCalculator.ComputeWarm(inputs, target == WarmRole.Active);
|
||||
ApplyWarmTransition(target, level, verdict.Reason);
|
||||
}
|
||||
|
||||
private WarmVerdict EvaluateWarmVerdict(NodeId self, MemberStatus memberState, bool listening)
|
||||
{
|
||||
NodeId? oldest = null, oldestReachable = null;
|
||||
var primaryUnreachable = false;
|
||||
var snapshotPrimary = _acceptedSnapshot?.Nodes.FirstOrDefault(n => n.Role == RedundancyRole.Primary)?.NodeId;
|
||||
try
|
||||
{
|
||||
var state = _cluster.State;
|
||||
var unreachable = state.Unreachable.ToHashSet();
|
||||
oldest = WarmRoleAuthority.OldestUpMember(state.Members, "driver");
|
||||
oldestReachable = WarmRoleAuthority.OldestUpMember(state.Members, "driver", unreachable);
|
||||
primaryUnreachable = snapshotPrimary is { } sp && unreachable.Any(m => WarmRoleAuthority.ToNodeId(m) == sp);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "OpcUaPublish: cluster state unavailable for the Warm verdict; treating membership as unknown");
|
||||
}
|
||||
|
||||
return WarmRoleAuthority.Evaluate(new WarmRoleInputs(
|
||||
self, memberState == MemberStatus.Up, listening, oldest, oldestReachable,
|
||||
HasAcceptedSnapshot: _acceptedSnapshot is not null, snapshotPrimary, primaryUnreachable,
|
||||
_redundancy.PromoteWhilePrimaryUnreachable));
|
||||
}
|
||||
|
||||
private void TrackDisagreement(WarmVerdict verdict, DateTime now)
|
||||
{
|
||||
if (!verdict.Disagreement)
|
||||
{
|
||||
_disagreeSinceUtc = null;
|
||||
_disagreementWarned = false;
|
||||
return;
|
||||
}
|
||||
|
||||
_disagreeSinceUtc ??= now;
|
||||
if (_disagreementWarned || now - _disagreeSinceUtc.Value <= WarmDisagreementWarning) return;
|
||||
_disagreementWarned = true;
|
||||
_log.Warning(
|
||||
"OpcUaPublish: Warm redundancy — cluster membership and the redundancy snapshot (revision {Revision}) have disagreed about this node being Primary for more than {Seconds}s; the node stays {Role}",
|
||||
_revisionGate.LastAccepted, WarmDisagreementWarning.TotalSeconds, _warmRole);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the data plane to <paramref name="target"/> in the spec §4.7 order and publishes
|
||||
/// <paramref name="level"/>. Invariant: an Active-band ServiceLevel is never visible while the projection is
|
||||
/// closed — a promotion publishes the level LAST and, if that publish fails, retries it; a demotion publishes
|
||||
/// the Standby level FIRST and degrades the data whether or not that publish succeeded (failing safe).
|
||||
/// </summary>
|
||||
private void ApplyWarmTransition(WarmRole target, byte level, string reason)
|
||||
{
|
||||
if (target == _warmRole)
|
||||
{
|
||||
if (!TryPublishServiceLevel(level) && target == WarmRole.Active) ScheduleWarmRetry();
|
||||
else if (target == WarmRole.Active) _promotionLevelPending = false;
|
||||
return;
|
||||
}
|
||||
|
||||
var projection = _standbyProjection;
|
||||
if (target == WarmRole.Active)
|
||||
{
|
||||
projection?.SetDataProjection(false); // 1. open the read projection
|
||||
projection?.QueueCurrentToLiveItems(false); // 2. current Good value to every live data item
|
||||
projection?.SetStandbyDelivery(false); // 3. first deliveries decided Active
|
||||
projection?.SetAlarmSuppression(false); // 4. new alarm events delivered
|
||||
projection?.RefreshAllEventItems(); // 5. promotion refresh on every live event item
|
||||
_warmRole = WarmRole.Active;
|
||||
if (TryPublishServiceLevel(level)) // 6. Active ServiceLevel, last
|
||||
{
|
||||
_promotionLevelPending = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
_promotionLevelPending = true;
|
||||
ScheduleWarmRetry();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
TryPublishServiceLevel(level); // 1. Standby ServiceLevel, first (continue on failure)
|
||||
projection?.SetStandbyDelivery(true); // 2. the D4 boundary
|
||||
projection?.SetAlarmSuppression(true); // 3. suppress new alarm events
|
||||
projection?.SetDataProjection(true); // 4. close the read projection
|
||||
projection?.QueueCurrentToLiveItems(true); // 5. BadOutOfService to every live data item
|
||||
_warmRole = WarmRole.Standby;
|
||||
_promotionLevelPending = false;
|
||||
}
|
||||
|
||||
OtOpcUaTelemetry.SetWarmRole(_warmRole == WarmRole.Active ? 1 : 0);
|
||||
OtOpcUaTelemetry.WarmTransition.Add(1,
|
||||
new KeyValuePair<string, object?>("direction", target == WarmRole.Active ? "promote" : "demote"),
|
||||
new KeyValuePair<string, object?>("reason", reason));
|
||||
_log.Info(
|
||||
"OpcUaPublish: Warm redundancy {Direction} to {Role} (reason={Reason}, revision={Revision}, ServiceLevel={Level})",
|
||||
target == WarmRole.Active ? "promotion" : "demotion", _warmRole, reason, _revisionGate.LastAccepted, level);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §3 deploy-time cross-check: under <c>None</c> or <c>Warm</c>, an artifact whose cluster
|
||||
/// <c>RedundancyMode</c> disagrees with the local mode (<c>Hot</c> included, owner answer Q5) logs an Error and
|
||||
/// counts <c>otopcua.redundancy.mode_mismatch</c>. The node keeps its local mode — the only value available
|
||||
/// before SDK start. <c>Unset</c> has opted out and is not checked.
|
||||
/// </summary>
|
||||
/// <param name="artifact">The applied artifact bytes.</param>
|
||||
private void CheckRedundancyMode(byte[] artifact)
|
||||
{
|
||||
if (_redundancy.Mode == OpcUaRedundancyMode.Unset || _localNode is null) return;
|
||||
string? clusterMode;
|
||||
try
|
||||
{
|
||||
clusterMode = DeploymentArtifact.ReadClusterRedundancyMode(artifact, _localNode.Value.Value);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "OpcUaPublish: could not read the artifact's cluster RedundancyMode");
|
||||
return;
|
||||
}
|
||||
|
||||
if (clusterMode is null) return;
|
||||
if (string.Equals(clusterMode, _redundancy.Mode.ToString(), StringComparison.OrdinalIgnoreCase)) return;
|
||||
OtOpcUaTelemetry.RedundancyModeMismatch.Add(1,
|
||||
new KeyValuePair<string, object?>("local", _redundancy.Mode.ToString()),
|
||||
new KeyValuePair<string, object?>("cluster", clusterMode));
|
||||
_log.Error(
|
||||
"OpcUaPublish: the deployed ServerCluster.RedundancyMode for this node's cluster is {ClusterMode} but OpcUa:Redundancy:Mode is {LocalMode}; the node keeps running as {LocalMode} (local configuration is the authority — fix whichever is wrong)",
|
||||
clusterMode, _redundancy.Mode, _redundancy.Mode);
|
||||
}
|
||||
|
||||
private void ScheduleWarmRetry() => Timers.StartSingleTimer("warm-retry", WarmRetry.Instance, TimeSpan.FromSeconds(1));
|
||||
|
||||
/// <summary>The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200,
|
||||
/// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
|
||||
private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
using Akka.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
|
||||
/// <summary>The Warm role (Gitea #717 spec §4.1).</summary>
|
||||
public enum WarmRole
|
||||
{
|
||||
/// <summary>Degraded standby: projected data, suppressed new alarm events, Degraded-band ServiceLevel.</summary>
|
||||
Standby,
|
||||
|
||||
/// <summary>Serves data and alarms; Healthy-band ServiceLevel.</summary>
|
||||
Active,
|
||||
}
|
||||
|
||||
/// <summary>The facts the Warm verdict is a pure function of.</summary>
|
||||
/// <param name="Self">This node's canonical <c>host:port</c> id.</param>
|
||||
/// <param name="SelfUp">This node's own member status is Up.</param>
|
||||
/// <param name="EndpointListening">This node's OPC UA endpoint did not fail to start (#762).</param>
|
||||
/// <param name="OldestUpDriver">The oldest Up <c>driver</c> member in this node's OWN cluster state (the singleton
|
||||
/// placement rule), or null when there is none.</param>
|
||||
/// <param name="OldestReachableUpDriver">The same, ignoring members this node sees as unreachable.</param>
|
||||
/// <param name="HasAcceptedSnapshot">A revisioned snapshot has been accepted (spec §4.2). Unrevisioned snapshots
|
||||
/// never count.</param>
|
||||
/// <param name="SnapshotPrimary">The Primary named by the latest accepted revisioned snapshot, or null.</param>
|
||||
/// <param name="SnapshotPrimaryUnreachable">This node sees <paramref name="SnapshotPrimary"/> as unreachable.</param>
|
||||
/// <param name="PromoteWhilePrimaryUnreachable">The owner-answer-Q2 option (default on).</param>
|
||||
public readonly record struct WarmRoleInputs(
|
||||
NodeId Self,
|
||||
bool SelfUp,
|
||||
bool EndpointListening,
|
||||
NodeId? OldestUpDriver,
|
||||
NodeId? OldestReachableUpDriver,
|
||||
bool HasAcceptedSnapshot,
|
||||
NodeId? SnapshotPrimary,
|
||||
bool SnapshotPrimaryUnreachable,
|
||||
bool PromoteWhilePrimaryUnreachable);
|
||||
|
||||
/// <summary>The verdict and why.</summary>
|
||||
/// <param name="Role">Active or Standby.</param>
|
||||
/// <param name="Reason">A short reason for the transition metric/log: <c>boot</c>, <c>endpoint</c>,
|
||||
/// <c>membership</c>, <c>snapshot</c>, <c>unreachable-primary</c> or <c>agreed</c>.</param>
|
||||
/// <param name="Disagreement">Membership (2) and snapshot (3) disagree about whether this node is Primary.</param>
|
||||
public readonly record struct WarmVerdict(WarmRole Role, string Reason, bool Disagreement);
|
||||
|
||||
/// <summary>
|
||||
/// The Warm role verdict (Gitea #717 spec §4.1), a pure function so every row is unit-testable. Active iff the
|
||||
/// node is Up and listening, its own cluster state names it the oldest Up driver member, and the latest accepted
|
||||
/// revisioned snapshot names it Primary. Anything else — no snapshot yet, an unrevisioned snapshot, a
|
||||
/// disagreement — is Standby. With <see cref="WarmRoleInputs.PromoteWhilePrimaryUnreachable"/> the survivor of a
|
||||
/// crashed Primary goes Active once that Primary is unreachable, before <c>auto-down</c> removes it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is a separate output from <c>PrimaryGatePolicy</c>, whose lone-node allow is unchanged for writes and
|
||||
/// acks (D7).
|
||||
/// </remarks>
|
||||
public static class WarmRoleAuthority
|
||||
{
|
||||
/// <summary>Evaluates the verdict.</summary>
|
||||
/// <param name="i">The inputs.</param>
|
||||
/// <returns>The verdict.</returns>
|
||||
public static WarmVerdict Evaluate(in WarmRoleInputs i)
|
||||
{
|
||||
if (!i.SelfUp) return new(WarmRole.Standby, "membership", false);
|
||||
if (!i.EndpointListening) return new(WarmRole.Standby, "endpoint", false);
|
||||
if (!i.HasAcceptedSnapshot) return new(WarmRole.Standby, "boot", false);
|
||||
|
||||
var membershipSaysSelf = i.OldestUpDriver == i.Self;
|
||||
var snapshotSaysSelf = i.SnapshotPrimary == i.Self;
|
||||
var disagreement = membershipSaysSelf != snapshotSaysSelf;
|
||||
|
||||
if (membershipSaysSelf && snapshotSaysSelf) return new(WarmRole.Active, "agreed", false);
|
||||
|
||||
// Owner answer Q2: a SIGKILLed Primary stays an Up-but-unreachable member until auto-down removes it, and the
|
||||
// singleton usually died with it. Evaluate (2) and (3) with unreachable members excluded.
|
||||
if (i.PromoteWhilePrimaryUnreachable
|
||||
&& i.OldestReachableUpDriver == i.Self
|
||||
&& (snapshotSaysSelf || (i.SnapshotPrimary is not null && i.SnapshotPrimaryUnreachable)))
|
||||
{
|
||||
return new(WarmRole.Active, "unreachable-primary", disagreement);
|
||||
}
|
||||
|
||||
return new(WarmRole.Standby, membershipSaysSelf ? "snapshot" : "membership", disagreement);
|
||||
}
|
||||
|
||||
/// <summary>The oldest Up member carrying <paramref name="role"/> (the singleton placement rule), optionally
|
||||
/// ignoring <paramref name="unreachable"/> members.</summary>
|
||||
/// <param name="members">The current members.</param>
|
||||
/// <param name="role">The role.</param>
|
||||
/// <param name="unreachable">Members to ignore, or null.</param>
|
||||
/// <returns>Its canonical node id, or null.</returns>
|
||||
public static NodeId? OldestUpMember(IEnumerable<Member> members, string role, IReadOnlySet<Member>? unreachable = null)
|
||||
{
|
||||
Member? oldest = null;
|
||||
foreach (var m in members)
|
||||
{
|
||||
if (m.Status != MemberStatus.Up || !m.HasRole(role)) continue;
|
||||
if (unreachable is not null && unreachable.Contains(m)) continue;
|
||||
if (oldest is null || m.IsOlderThan(oldest)) oldest = m;
|
||||
}
|
||||
|
||||
return oldest is null ? null : ToNodeId(oldest);
|
||||
}
|
||||
|
||||
/// <summary>The canonical <c>host:port</c> id of a member (the RedundancyStateActor format).</summary>
|
||||
/// <param name="m">The member.</param>
|
||||
/// <returns>The node id.</returns>
|
||||
public static NodeId ToNodeId(Member m) => NodeId.Parse($"{m.Address.Host ?? string.Empty}:{m.Address.Port ?? 0}");
|
||||
}
|
||||
@@ -566,6 +566,9 @@ public static class ServiceCollectionExtensions
|
||||
// Optional: a harness that builds this graph without the hosted service registers none, and both
|
||||
// consumers then behave as if the endpoint were listening (the pre-#762 behaviour).
|
||||
var endpointState = resolver.GetService<IOpcUaEndpointState>();
|
||||
// Gitea #717: OpcUa:Redundancy (registered by the Host from the validated OpcUa options). Absent in harnesses ⇒
|
||||
// Unset ⇒ today's behaviour. The standby projection is late-bound like the ServiceLevel publisher.
|
||||
var redundancy = resolver.GetService<WarmRedundancySettings>() ?? WarmRedundancySettings.Unset;
|
||||
var publishActor = system.ActorOf(
|
||||
OpcUaPublishActor.Props(
|
||||
sink: addressSpaceSink,
|
||||
@@ -576,7 +579,9 @@ public static class ServiceCollectionExtensions
|
||||
dbHealthProbe: dbHealth,
|
||||
dbLess: dbFactory is null,
|
||||
dataPlaneAuthority: dataPlaneAuthority,
|
||||
endpointState: endpointState),
|
||||
endpointState: endpointState,
|
||||
redundancy: redundancy,
|
||||
standbyProjection: redundancy.IsWarm ? resolver.GetService<DeferredStandbyProjection>() : null),
|
||||
OpcUaPublishActorName);
|
||||
registry.Register<OpcUaPublishActorKey>(publishActor);
|
||||
|
||||
@@ -670,7 +675,9 @@ public static class ServiceCollectionExtensions
|
||||
// lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
|
||||
// the InProcess default, when the seam is not listed, and in test graphs that register no provider.
|
||||
scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
|
||||
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
|
||||
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm),
|
||||
// Gitea #717 spec §4.2: drop older redundancy snapshots under Warm only.
|
||||
enforceSnapshotOrdering: redundancy.IsWarm),
|
||||
DriverHostActorName);
|
||||
registry.Register<DriverHostActorKey>(driverHost);
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.2 (ordered snapshots) and §4.3 (Warm bands): the revision the singleton stamps, its order,
|
||||
/// the subscriber gate, the wire shape of the extended message, and the Warm ServiceLevel truth table.
|
||||
/// </summary>
|
||||
public sealed class WarmRedundancyRevisionTests : ControlPlaneActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void The_singleton_stamps_an_increasing_revision_on_every_publish_heartbeats_included()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
Sys.ActorOf(RedundancyStateActor.Props(msg => probe.Ref.Tell(msg), TimeSpan.FromMilliseconds(200)));
|
||||
var first = probe.ExpectMsg<RedundancyStateChanged>(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);
|
||||
var second = probe.ExpectMsg<RedundancyStateChanged>(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
first.Revision.ShouldNotBeNull();
|
||||
second.Revision.ShouldNotBeNull();
|
||||
second.Revision!.CompareTo(first.Revision).ShouldBeGreaterThan(0);
|
||||
second.Revision.Sequence.ShouldBe(first.Revision!.Sequence + 1);
|
||||
first.Revision.PublisherUpNumber.ShouldBeGreaterThan(0L, "the publisher's Akka up-number");
|
||||
}
|
||||
|
||||
/// <summary>Akka.NET keeps <c>Member.UpNumber</c> internal; an upgrade that renames it must fail here, not degrade
|
||||
/// every revision to up-number 0 silently.</summary>
|
||||
[Fact]
|
||||
public void Akka_member_up_number_is_readable()
|
||||
{
|
||||
AwaitCondition(() => Akka.Cluster.Cluster.Get(Sys).SelfMember.Status == MemberStatus.Up, TimeSpan.FromSeconds(10),
|
||||
cancellationToken: TestContext.Current.CancellationToken);
|
||||
RedundancyStateActor.ReadUpNumber(Akka.Cluster.Cluster.Get(Sys).SelfMember).ShouldNotBeNull().ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_singleton_restarted_in_the_same_process_outranks_its_predecessor()
|
||||
{
|
||||
var a = RedundancyStateActor.NextIncarnationTick();
|
||||
var b = RedundancyStateActor.NextIncarnationTick();
|
||||
b.ShouldBeGreaterThan(a, "strictly increasing even within one tick");
|
||||
new SnapshotRevision(1, b, 1).CompareTo(new SnapshotRevision(1, a, 500)).ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Revisions_order_by_up_number_then_incarnation_then_sequence()
|
||||
{
|
||||
var baseline = new SnapshotRevision(2, 100, 50);
|
||||
new SnapshotRevision(3, 1, 1).CompareTo(baseline).ShouldBeGreaterThan(0, "a handover to a younger member outranks");
|
||||
new SnapshotRevision(1, 999, 999).CompareTo(baseline).ShouldBeLessThan(0, "a departed older member's late publish loses");
|
||||
new SnapshotRevision(2, 101, 1).CompareTo(baseline).ShouldBeGreaterThan(0);
|
||||
new SnapshotRevision(2, 100, 51).CompareTo(baseline).ShouldBeGreaterThan(0);
|
||||
new SnapshotRevision(2, 100, 50).CompareTo(baseline).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_gate_accepts_only_newer_revisions_and_never_advances_on_an_unrevisioned_snapshot()
|
||||
{
|
||||
var gate = new SnapshotRevisionGate();
|
||||
gate.Offer(Snap(null)).ShouldBe(SnapshotRejection.Unrevisioned);
|
||||
gate.LastAccepted.ShouldBeNull();
|
||||
gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.None);
|
||||
gate.Offer(Snap(new(1, 9, 9))).ShouldBe(SnapshotRejection.OlderRevision, "an old incarnation's late heartbeat");
|
||||
gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.OlderRevision, "a duplicate");
|
||||
gate.Offer(Snap(new(2, 1, 6))).ShouldBe(SnapshotRejection.None);
|
||||
gate.LastAccepted.ShouldBe(new SnapshotRevision(2, 1, 6));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_extended_message_round_trips_and_a_pre_717_payload_decodes_with_no_revision()
|
||||
{
|
||||
var msg = Snap(new SnapshotRevision(3, 42, 7));
|
||||
var serializer = Sys.Serialization.FindSerializerFor(msg);
|
||||
var manifest = Akka.Serialization.Serialization.ManifestFor(serializer, msg);
|
||||
var round = (RedundancyStateChanged)Sys.Serialization.Deserialize(serializer.ToBinary(msg), serializer.Identifier, manifest);
|
||||
round.Revision.ShouldBe(msg.Revision);
|
||||
round.Nodes.Single().NodeId.ShouldBe(msg.Nodes.Single().NodeId);
|
||||
|
||||
// A pre-#717 publisher's payload: the same message without the field.
|
||||
var legacy = Snap(null);
|
||||
var legacyBytes = serializer.ToBinary(legacy);
|
||||
var text = System.Text.Encoding.UTF8.GetString(legacyBytes);
|
||||
var stripped = System.Text.Encoding.UTF8.GetBytes(text.Replace(",\"Revision\":null", string.Empty, StringComparison.Ordinal));
|
||||
var decoded = (RedundancyStateChanged)Sys.Serialization.Deserialize(stripped, serializer.Identifier, manifest);
|
||||
decoded.Revision.ShouldBeNull();
|
||||
decoded.Nodes.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
private static RedundancyStateChanged Snap(SnapshotRevision? revision) =>
|
||||
new(new[] { new NodeRedundancyState(NodeId.Parse("a:1"), RedundancyRole.Primary, true, true, DateTime.UtcNow) },
|
||||
CorrelationId.NewId(), revision);
|
||||
}
|
||||
|
||||
/// <summary>Gitea #717 spec §4.3: the Warm ServiceLevel bands (pure; no actor system).</summary>
|
||||
public sealed class WarmServiceLevelBandTests
|
||||
{
|
||||
|
||||
public static IEnumerable<object[]> AllInputs()
|
||||
{
|
||||
foreach (var member in new[] { MemberStatus.Up, MemberStatus.Joining, MemberStatus.Leaving, MemberStatus.Down })
|
||||
foreach (var db in new[] { true, false })
|
||||
foreach (var probe in new[] { true, false })
|
||||
foreach (var stale in new[] { true, false })
|
||||
foreach (var listening in new[] { true, false })
|
||||
yield return [member, db, probe, stale, listening];
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(AllInputs))]
|
||||
public void Every_standby_level_is_below_every_active_level_that_serves_data(
|
||||
MemberStatus member, bool db, bool probe, bool stale, bool listening)
|
||||
{
|
||||
var h = new NodeHealthInputs(member, db, probe, stale, IsDriverPrimary: false, listening);
|
||||
var standby = ServiceLevelCalculator.ComputeWarm(h, isActive: false);
|
||||
var active = ServiceLevelCalculator.ComputeWarm(h, isActive: true);
|
||||
standby.ShouldBeLessThanOrEqualTo(ServiceLevelCalculator.WarmStandbyHealthy);
|
||||
if (active > 0) active.ShouldBeGreaterThanOrEqualTo(ServiceLevelCalculator.WarmActiveCritical);
|
||||
if (active > 0) standby.ShouldBeLessThan(active);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, true, false, true, 250)]
|
||||
[InlineData(true, true, true, true, 200)]
|
||||
[InlineData(false, true, true, true, 100)]
|
||||
[InlineData(false, true, false, true, 100)]
|
||||
[InlineData(true, false, false, true, 100)]
|
||||
[InlineData(true, true, false, false, 90)]
|
||||
[InlineData(true, true, true, false, 50)]
|
||||
[InlineData(false, true, false, false, 50)]
|
||||
[InlineData(true, false, false, false, 50)]
|
||||
public void Warm_bands_match_the_spec_table(bool db, bool probe, bool stale, bool active, int expected)
|
||||
{
|
||||
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, db, probe, stale, IsDriverPrimary: false), active)
|
||||
.ShouldBe((byte)expected);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void Down_or_a_failed_endpoint_is_zero_in_either_role(bool active)
|
||||
{
|
||||
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Down, true, true, false, false), active).ShouldBe((byte)0);
|
||||
ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, true, true, false, false, EndpointListening: false), active)
|
||||
.ShouldBe((byte)0);
|
||||
}
|
||||
}
|
||||
+35
@@ -132,3 +132,38 @@ public sealed class OpcUaApplicationHostOptionsValidatorTests
|
||||
if (!valid) result.Failures!.ShouldContain(f => f.Contains("OpcUa:Discovery"));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Gitea #717 spec §3: the redundancy rules the startup validator and the driver-role guard enforce.</summary>
|
||||
public sealed class OpcUaRedundancyValidationTests
|
||||
{
|
||||
private static readonly OpcUaApplicationHostOptionsValidator Sut = new();
|
||||
|
||||
[Theory]
|
||||
[InlineData("Unset", new string[0], true)]
|
||||
[InlineData("None", new string[0], true)]
|
||||
[InlineData("None", new[] { "urn:peer" }, false)]
|
||||
[InlineData("Warm", new[] { "urn:peer" }, true)]
|
||||
[InlineData("Warm", new string[0], false)]
|
||||
[InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)]
|
||||
[InlineData("Warm", new[] { "urn:OtOpcUa" }, false)]
|
||||
[InlineData("Hot", new[] { "urn:peer" }, false)]
|
||||
[InlineData("Bogus", new string[0], false)]
|
||||
public void Redundancy_matrix(string mode, string[] peers, bool valid)
|
||||
{
|
||||
var result = Sut.Validate(null, new OpcUaApplicationHostOptions
|
||||
{
|
||||
Redundancy = new OpcUaRedundancyOptions { Mode = mode },
|
||||
PeerApplicationUris = peers.ToList(),
|
||||
});
|
||||
result.Succeeded.ShouldBe(valid, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Warm_on_a_node_without_the_driver_role_fails_start()
|
||||
{
|
||||
Should.Throw<InvalidOperationException>(() => OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: false));
|
||||
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: true);
|
||||
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(null, hasDriver: false);
|
||||
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("None", hasDriver: false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,9 @@ internal sealed class ServerEventRig : IAsyncDisposable
|
||||
|
||||
public string EndpointUrl => $"opc.tcp://127.0.0.1:{_port}/OtOpcUa";
|
||||
|
||||
/// <summary>The configured <c>OpcUa:ApplicationUri</c> (the effective one: the rig loads no XML).</summary>
|
||||
public string ApplicationUri => _hostOptions.ApplicationUri;
|
||||
|
||||
public OtOpcUaNodeManager Nm => _server.NodeManager!;
|
||||
|
||||
internal OtOpcUaSdkServer Server => _server;
|
||||
@@ -57,7 +60,12 @@ internal sealed class ServerEventRig : IAsyncDisposable
|
||||
|
||||
public SdkAddressSpaceSink Sink { get; private set; } = null!;
|
||||
|
||||
public static async Task<ServerEventRig> StartAsync(string name, bool encrypted, CancellationToken ct)
|
||||
public static Task<ServerEventRig> StartAsync(string name, bool encrypted, CancellationToken ct) =>
|
||||
StartAsync(name, encrypted, ct, configure: null);
|
||||
|
||||
/// <summary>Starts the rig, letting the caller adjust the host options first (Gitea #717: redundancy mode, peers).</summary>
|
||||
public static async Task<ServerEventRig> StartAsync(
|
||||
string name, bool encrypted, CancellationToken ct, Action<OpcUaApplicationHostOptions>? configure)
|
||||
{
|
||||
var rig = new ServerEventRig(Path.Combine(Path.GetTempPath(), $"otopcua-{name}-{Guid.NewGuid():N}"));
|
||||
rig._hostOptions = new OpcUaApplicationHostOptions
|
||||
@@ -71,6 +79,7 @@ internal sealed class ServerEventRig : IAsyncDisposable
|
||||
: [OpcUaSecurityProfile.None],
|
||||
AutoAcceptUntrustedClientCertificates = true,
|
||||
};
|
||||
configure?.Invoke(rig._hostOptions);
|
||||
(rig._host, rig._server) = await OpcUaHostStart.StartAsync(
|
||||
port =>
|
||||
{
|
||||
|
||||
+616
@@ -0,0 +1,616 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 over the wire, against the real <see cref="OtOpcUaSdkServer"/> and node manager: the redundancy
|
||||
/// metadata per mode (T2), the Standby read projection and write refusal (T6), the D4 first-delivery decision record
|
||||
/// across the three demotion timings T0 established (T7), the live-item queue on promotion/demotion (T8), and the
|
||||
/// alarm suppression, refusal and promotion refresh (T9). Transitions are driven through the server's own
|
||||
/// <see cref="IStandbyProjection"/> in the spec §4.7 step order; the actor that owns that order has its own tests.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class WarmRedundancyWireTests
|
||||
{
|
||||
private const string Peer = "urn:OtOpcUa:peer-717";
|
||||
|
||||
private const NodePermissions All =
|
||||
NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead
|
||||
| NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve
|
||||
| NodePermissions.MethodCall | NodePermissions.WriteOperate;
|
||||
|
||||
private static Task<ServerEventRig> StartModeAsync(string name, string mode, CancellationToken ct) =>
|
||||
ServerEventRig.StartAsync(name, encrypted: false, ct, o =>
|
||||
{
|
||||
o.Redundancy = new OpcUaRedundancyOptions { Mode = mode };
|
||||
if (mode == "Warm") o.PeerApplicationUris = [Peer];
|
||||
});
|
||||
|
||||
/// <summary>Spec §4.7 promotion, in order (the actor's sequence, minus the ServiceLevel publish).</summary>
|
||||
internal static void Promote(IStandbyProjection p)
|
||||
{
|
||||
p.SetDataProjection(false);
|
||||
p.QueueCurrentToLiveItems(false);
|
||||
p.SetStandbyDelivery(false);
|
||||
p.SetAlarmSuppression(false);
|
||||
p.RefreshAllEventItems();
|
||||
}
|
||||
|
||||
/// <summary>Spec §4.7 demotion, in order (after the Standby ServiceLevel publish).</summary>
|
||||
internal static void Demote(IStandbyProjection p)
|
||||
{
|
||||
p.SetStandbyDelivery(true);
|
||||
p.SetAlarmSuppression(true);
|
||||
p.SetDataProjection(true);
|
||||
p.QueueCurrentToLiveItems(true);
|
||||
}
|
||||
|
||||
private static CancellationToken Deadline(int seconds = 90)
|
||||
{
|
||||
var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
|
||||
cts.CancelAfter(TimeSpan.FromSeconds(seconds));
|
||||
return cts.Token;
|
||||
}
|
||||
|
||||
private static RequestHeader H() => new() { Timestamp = DateTimeUtc.Now, TimeoutHint = 10_000 };
|
||||
|
||||
private static NodeId Raw(ISession session, string id) =>
|
||||
new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
|
||||
|
||||
private static async Task<DataValue> ReadAsync(ISession session, NodeId node, CancellationToken ct, uint attribute = Attributes.Value,
|
||||
string? indexRange = null)
|
||||
{
|
||||
var r = await ((SessionClient)session).ReadAsync(H(), 0, TimestampsToReturn.Both,
|
||||
new[] { new ReadValueId { NodeId = node, AttributeId = attribute, IndexRange = indexRange } }, ct);
|
||||
return r.Results[0];
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ T2 metadata
|
||||
|
||||
[Fact]
|
||||
public async Task Warm_advertises_non_transparent_metadata_self_first_and_boots_at_ServiceLevel_1()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717Meta", "Warm", ct);
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
|
||||
Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
|
||||
.ShouldBe((int)RedundancySupport.Warm);
|
||||
var uris = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
|
||||
uris.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
((string[])uris.ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
|
||||
((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
|
||||
Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.WarmBootServiceLevel);
|
||||
|
||||
var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
|
||||
{
|
||||
new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
|
||||
ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
|
||||
}, ct);
|
||||
browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.NonTransparentRedundancyType));
|
||||
|
||||
var write = await ((SessionClient)session).WriteAsync(H(), new[]
|
||||
{
|
||||
new WriteValue { NodeId = VariableIds.Server_ServerRedundancy_ServerUriArray, AttributeId = Attributes.Value,
|
||||
Value = new DataValue(new Variant(new[] { "x" })) },
|
||||
}, ct);
|
||||
StatusCode.IsBad(write.Results[0]).ShouldBeTrue($"ServerUriArray is read-only, got {write.Results[0]}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task None_advertises_RedundancySupport_None_and_ServerUriArray_is_absent_even_by_NodeId()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717None", "None", ct);
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
|
||||
Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
|
||||
.ShouldBe((int)RedundancySupport.None);
|
||||
(await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct)).StatusCode
|
||||
.ShouldBe(StatusCodes.BadNodeIdUnknown, "T0a: the SDK's orphan i=11314 is deleted under None");
|
||||
var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
|
||||
{
|
||||
new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
|
||||
ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
|
||||
}, ct);
|
||||
browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.ServerRedundancyType));
|
||||
((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri });
|
||||
}
|
||||
|
||||
/// <summary>The default-off pin: an Unset server is today's server — SDK-default RedundancySupport, the SDK's own
|
||||
/// (orphan) i=11314, ServerRedundancyType, the #885 boot level 0, Good data with no projection, and no Warm hooks.</summary>
|
||||
[Fact]
|
||||
public async Task Unset_is_todays_server_exactly()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await ServerEventRig.StartAsync("Warm717Unset", encrypted: false, ct);
|
||||
rig.Server.RedundancyMode.ShouldBe(OpcUaRedundancyMode.Unset);
|
||||
rig.Server.CreateStandbyProjection().ShouldBeNull();
|
||||
rig.Nm.Standby.WarmMode.ShouldBeFalse();
|
||||
var id = rig.EnsureRawVariable("u", "Double", writable: false);
|
||||
rig.Sink.WriteValue(id, 4.5, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var support = await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct);
|
||||
support.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
Convert.ToInt32(support.ClrValue()).ShouldBe((int)RedundancySupport.None, "the SDK default, never written by us");
|
||||
var orphan = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
|
||||
orphan.StatusCode.ShouldBe(StatusCodes.Good, "Unset leaves the SDK's orphan i=11314 exactly as the SDK builds it");
|
||||
Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel);
|
||||
var value = await ReadAsync(session, Raw(session, id), ct);
|
||||
value.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
value.ClrValue().ShouldBe(4.5);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ T6 read projection + write refusal
|
||||
|
||||
[Fact]
|
||||
public async Task Standby_projects_ordinary_tag_reads_but_not_metadata_and_promotion_reads_the_latest_stored_value()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717Read", "Warm", ct);
|
||||
var projection = rig.Server.CreateStandbyProjection().ShouldNotBeNull();
|
||||
var raw = rig.EnsureRawVariable("t1", "Double", writable: false);
|
||||
rig.Sink.EnsureVariable("EqA/t1ref", null, "t1ref", "Double", writable: false, AddressSpaceRealm.Uns);
|
||||
var arr = $"{RawDeviceFolder}/arr";
|
||||
rig.Sink.EnsureVariable(arr, RawDeviceFolder, "arr", "Int32", writable: false, AddressSpaceRealm.Raw, isArray: true);
|
||||
rig.EnsureEquipment();
|
||||
rig.Sink.EnsureVariable("EqA/vt", "EqA", "vt", "Double", writable: false, AddressSpaceRealm.Uns);
|
||||
rig.MaterialiseScripted();
|
||||
var ts = DateTime.UtcNow;
|
||||
rig.Sink.WriteValue(raw, 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
|
||||
rig.Sink.WriteValue("EqA/t1ref", 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
|
||||
rig.Sink.WriteValue("EqA/vt", 2.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
|
||||
rig.Sink.WriteValue(arr, new[] { 1, 2, 3 }, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
foreach (var node in new[] { Raw(session, raw), UnsNodeId(session, "EqA/t1ref"), UnsNodeId(session, "EqA/vt") })
|
||||
{
|
||||
var dv = await ReadAsync(session, node, ct);
|
||||
dv.StatusCode.ShouldBe(StatusCodes.BadOutOfService, node.ToString());
|
||||
dv.WrappedValue.IsNull.ShouldBeTrue();
|
||||
(dv.ServerTimestamp != DateTimeUtc.MinValue).ShouldBeTrue();
|
||||
}
|
||||
|
||||
(await ReadAsync(session, Raw(session, arr), ct, indexRange: "1")).StatusCode.ShouldBe(StatusCodes.BadOutOfService);
|
||||
(await ReadAsync(session, Raw(session, raw), ct, Attributes.DisplayName)).StatusCode.ShouldBe(StatusCodes.Good);
|
||||
(await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).StatusCode.ShouldBe(StatusCodes.Good);
|
||||
(await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).StatusCode.ShouldBe(StatusCodes.Good);
|
||||
var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
|
||||
(await ReadAsync(session, severity, ct)).StatusCode.ShouldBe(StatusCodes.Good, "a condition child is not an ordinary tag");
|
||||
|
||||
// Acquisition continues while projected; promotion reads the latest stored value with no new driver publish.
|
||||
rig.Sink.WriteValue(raw, 9.25, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
Promote(projection);
|
||||
var after = await ReadAsync(session, Raw(session, raw), ct);
|
||||
after.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
after.ClrValue().ShouldBe(9.25);
|
||||
var ranged = await ReadAsync(session, Raw(session, arr), ct, indexRange: "1");
|
||||
ranged.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
((int[])ranged.ClrValue()!).ShouldBe(new[] { 2 }, "an IndexRange read reaches the stored array once promoted");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Standby_projection_never_leaks_an_ACL_denied_node()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717Acl", "Warm", ct);
|
||||
var raw = rig.EnsureRawVariable("secret", "Double", writable: false);
|
||||
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
rig.Enforce(true, [Key()]); // anonymous holds no grant
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
(await ReadAsync(session, Raw(session, raw), ct)).StatusCode.ShouldBe(StatusCodes.BadUserAccessDenied);
|
||||
}
|
||||
|
||||
private sealed class RecordingWriteGateway : IOpcUaNodeWriteGateway
|
||||
{
|
||||
public int Count;
|
||||
|
||||
public Task<NodeWriteOutcome> WriteAsync(string nodeId, object? value, AddressSpaceRealm realm, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref Count);
|
||||
return Task.FromResult(new NodeWriteOutcome(true, null));
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Warm", true)]
|
||||
[InlineData("Warm", false)]
|
||||
[InlineData("Unset", true)]
|
||||
public async Task A_write_on_a_Warm_standby_is_refused_BadInvalidState_and_never_reaches_the_gateway(string mode, bool standby)
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync($"Warm717Write{mode}{standby}", mode, ct);
|
||||
var gateway = new RecordingWriteGateway();
|
||||
rig.Nm.NodeWriteGateway = gateway;
|
||||
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
|
||||
var raw = rig.EnsureRawVariable("w", "Double", writable: true);
|
||||
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
if (mode == "Warm" && !standby) Promote(rig.Server.CreateStandbyProjection()!);
|
||||
// Unset with "standby": a node that is not the Primary takes today's optimistic path (no gate wired here).
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var result = await ((SessionClient)session).WriteAsync(H(), new[]
|
||||
{
|
||||
new WriteValue { NodeId = Raw(session, raw), AttributeId = Attributes.Value, Value = new DataValue(new Variant(2.0)) },
|
||||
}, ct);
|
||||
|
||||
if (mode == "Warm" && standby)
|
||||
{
|
||||
result.Results[0].ShouldBe(StatusCodes.BadInvalidState);
|
||||
await Task.Delay(200, ct);
|
||||
gateway.Count.ShouldBe(0, "a refused write never reaches the driver");
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Results[0].ShouldBe(StatusCodes.Good);
|
||||
await RecordedEvents.WaitUntilAsync(() => Volatile.Read(ref gateway.Count) == 1);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ T9 Part 9 method refusal
|
||||
|
||||
[Theory]
|
||||
[InlineData("Warm", true, "Acknowledge")]
|
||||
[InlineData("Warm", true, "OneShotShelve")]
|
||||
[InlineData("Warm", false, "Acknowledge")]
|
||||
[InlineData("Unset", true, "Acknowledge")]
|
||||
public async Task A_Part9_method_on_a_Warm_standby_is_refused_BadInvalidState_before_any_state_change(
|
||||
string mode, bool standby, string method)
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync($"Warm717Method{mode}{standby}{method}", mode, ct);
|
||||
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
|
||||
rig.MaterialiseScripted();
|
||||
var routed = 0;
|
||||
rig.Nm.AlarmCommandRouter = _ => Interlocked.Increment(ref routed);
|
||||
var projection = rig.Server.CreateStandbyProjection();
|
||||
if (projection is not null) Promote(projection); // project the condition while Active so it has an EventId
|
||||
rig.Project(native: false, ConditionMethodEventTests.PreState(method, native: false) with { Message = "base" });
|
||||
if (projection is not null && standby) Demote(projection);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var eventId = await ReadEventIdAsync(session, ct);
|
||||
var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), method, eventId, ct);
|
||||
if (mode == "Warm" && standby)
|
||||
{
|
||||
(await Should.ThrowAsync<ServiceResultException>(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState);
|
||||
routed.ShouldBe(0, "nothing is routed to the engine");
|
||||
var acked = await TranslateAsync(session, ConditionNodeId(session, native: false), "AckedState/Id", ct);
|
||||
(await ReadAsync(session, acked, ct)).ClrValue().ShouldBe(false, "no condition state changed");
|
||||
}
|
||||
else
|
||||
{
|
||||
await call;
|
||||
routed.ShouldBe(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<byte[]> ReadEventIdAsync(ISession session, CancellationToken ct)
|
||||
{
|
||||
var eventId = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.EventId, ct);
|
||||
return ((ByteString)(await ReadAsync(session, eventId, ct)).WrappedValue.AsBoxedObject()!).ToArray()!;
|
||||
}
|
||||
|
||||
private static async Task<NodeId> TranslateAsync(ISession session, NodeId start, string path, CancellationToken ct)
|
||||
{
|
||||
var elements = path.Split('/').Select(name => new RelativePathElement
|
||||
{
|
||||
ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, TargetName = new QualifiedName(name),
|
||||
}).ToArray();
|
||||
var r = await ((SessionClient)session).TranslateBrowsePathsToNodeIdsAsync(H(), new[]
|
||||
{
|
||||
new BrowsePath { StartingNode = start, RelativePath = new RelativePath { Elements = elements } },
|
||||
}, ct);
|
||||
r.Results[0].StatusCode.ShouldBe(StatusCodes.Good, path);
|
||||
return ExpandedNodeId.ToNodeId(r.Results[0].Targets[0].TargetId, session.NamespaceUris);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ T9 alarm suppression + refresh
|
||||
|
||||
[Fact]
|
||||
public async Task Standby_suppresses_new_alarm_events_on_every_item_kind_and_promotion_refreshes_them_all()
|
||||
{
|
||||
var ct = Deadline(120);
|
||||
await using var rig = await StartModeAsync("Warm717Alarms", "Warm", ct);
|
||||
var projection = rig.Server.CreateStandbyProjection()!;
|
||||
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
|
||||
rig.MaterialiseNative();
|
||||
rig.MaterialiseScripted();
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
|
||||
var folderItem = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA));
|
||||
var deviceItem = RecordedEvents.AddEventItem(subscription,
|
||||
new NodeId(RawDeviceFolder, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri)));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
// While Standby (boot state): a new transition reaches no item, but the condition node is current.
|
||||
rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
|
||||
Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "while-standby"));
|
||||
// A condition raised and cleared (Retain=false) while Standby is not replayed by the promotion refresh.
|
||||
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
|
||||
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash"));
|
||||
rig.Project(native: false, new AlarmConditionSnapshot(Active: false, Acknowledged: true, Confirmed: true, Enabled: true,
|
||||
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash-cleared"));
|
||||
await Task.Delay(1000, ct);
|
||||
foreach (var item in new[] { serverItem, folderItem, deviceItem })
|
||||
{
|
||||
events.Has(item, "while-standby").ShouldBeFalse($"no alarm event on {item.StartNodeId} while Standby");
|
||||
events.Has(item, "flash").ShouldBeFalse();
|
||||
}
|
||||
|
||||
// D5: a client ConditionRefresh on the Standby returns Start, the retained condition, End.
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
|
||||
events.Has(serverItem, "while-standby").ShouldBeTrue("the Standby's refresh carries the retained condition");
|
||||
events.Has(serverItem, "flash-cleared").ShouldBeFalse("a non-retained condition is not replayed");
|
||||
foreach (var item in new[] { folderItem, deviceItem })
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
|
||||
var countsBefore = new[] { serverItem, folderItem, deviceItem }.ToDictionary(i => i, i => events.Of(i).Count);
|
||||
|
||||
Promote(projection);
|
||||
foreach (var item in new[] { serverItem, folderItem, deviceItem })
|
||||
{
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Of(item).Skip(countsBefore[item])
|
||||
.Any(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType));
|
||||
var stream = events.Of(item).Skip(countsBefore[item]).ToList();
|
||||
var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
|
||||
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
||||
start.ShouldBeGreaterThanOrEqualTo(0, $"RefreshStart on {item.StartNodeId}");
|
||||
var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
|
||||
replayed.ShouldContain("while-standby", $"promotion refresh replays the retained condition on {item.StartNodeId}");
|
||||
replayed.ShouldNotContain("flash-cleared");
|
||||
}
|
||||
|
||||
// After promotion new events flow, and the Standby-stamped EventId acknowledges.
|
||||
rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
|
||||
Shelving: AlarmShelvingKind.Unshelved, Severity: 800, Message: "after"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "after") && events.Has(folderItem, "after") && events.Has(deviceItem, "after"));
|
||||
await subscription.DeleteAsync(true, ct);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_Standby_stamped_EventId_is_accepted_by_Acknowledge_after_promotion()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717AckAfter", "Warm", ct);
|
||||
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
|
||||
rig.MaterialiseScripted();
|
||||
var routed = new List<AlarmCommand>();
|
||||
rig.Nm.AlarmCommandRouter = cmd => { lock (routed) routed.Add(cmd); };
|
||||
rig.Project(native: false, ConditionMethodEventTests.PreState("Acknowledge", native: false) with { Message = "stamped" });
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var stamped = await ReadEventIdAsync(session, ct); // stamped while Standby, never delivered
|
||||
Promote(rig.Server.CreateStandbyProjection()!);
|
||||
await ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), "Acknowledge", stamped, ct);
|
||||
lock (routed) routed.ShouldHaveSingleItem().Operation.ShouldBe("Acknowledge");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ T8 live items on transition
|
||||
|
||||
[Fact]
|
||||
public async Task Demotion_queues_one_BadOutOfService_then_silence_and_promotion_queues_the_current_value_through_a_deadband()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717Live", "Warm", ct);
|
||||
var projection = rig.Server.CreateStandbyProjection()!;
|
||||
var raw = rig.EnsureRawVariable("live", "Double", writable: false);
|
||||
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
Promote(projection);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var client = (SessionClient)session;
|
||||
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
|
||||
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
|
||||
await AddItemAsync(client, sub, Raw(session, raw), 2, 10, ct, new DataChangeFilter
|
||||
{
|
||||
Trigger = DataChangeTrigger.StatusValue, DeadbandType = (uint)DeadbandType.Absolute, DeadbandValue = 1000,
|
||||
});
|
||||
(await CollectAsync(client, ct, TimeSpan.FromSeconds(1))).Where(x => x.Handle is 1 or 2).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
|
||||
|
||||
Demote(projection);
|
||||
rig.Sink.WriteValue(raw, 2.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
rig.Sink.WriteValue(raw, 3.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
var demoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
|
||||
demoted.Count(x => x.Handle == 1).ShouldBe(1, "exactly one notification per item, then silence");
|
||||
demoted.Count(x => x.Handle == 2).ShouldBe(1);
|
||||
demoted.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
|
||||
|
||||
rig.Sink.WriteValue(raw, 4.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
Promote(projection);
|
||||
var promoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
|
||||
promoted.ShouldContain(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0));
|
||||
promoted.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0),
|
||||
"the promotion value is queued ignoring the deadband");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ T7 first-delivery decision record (D4)
|
||||
|
||||
/// <summary>Timing 1 (T0): Good samples queued in the item while Active, a client-handle change, then demotion.</summary>
|
||||
[Fact]
|
||||
public async Task Timing1_queued_good_samples_then_demotion_first_delivery_is_projected_and_republish_matches()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717T1", "Warm", ct);
|
||||
var projection = rig.Server.CreateStandbyProjection()!;
|
||||
var raw = rig.EnsureRawVariable("q", "Double", writable: false);
|
||||
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
rig.MaterialiseScripted();
|
||||
Promote(projection);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var client = (SessionClient)session;
|
||||
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
|
||||
var tagItem = await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
|
||||
await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct);
|
||||
var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
|
||||
await AddItemAsync(client, sub, severity, 3, 1, ct);
|
||||
(await PublishDataAsync(client, ct)).NotificationMessage.NotificationData.Count.ShouldBeGreaterThan(0);
|
||||
|
||||
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
|
||||
rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
// Non-ordinary changes queued in the same window: i=2267 and a condition child (Severity).
|
||||
new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
|
||||
Microsoft.Extensions.Logging.Abstractions.NullLogger<SdkServiceLevelPublisher>.Instance).Publish(90);
|
||||
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
|
||||
Shelving: AlarmShelvingKind.Unshelved, Severity: 650, Message: "child"));
|
||||
await client.ModifyMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
|
||||
{
|
||||
new MonitoredItemModifyRequest { MonitoredItemId = tagItem,
|
||||
RequestedParameters = new MonitoringParameters { ClientHandle = 99, SamplingInterval = 0, QueueSize = 10, DiscardOldest = true } },
|
||||
}, ct);
|
||||
DemoteDeliveryOnly(projection);
|
||||
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
|
||||
var msg = await PublishDataAsync(client, ct);
|
||||
var changes = DataChanges(msg.NotificationMessage);
|
||||
changes.Where(x => x.Handle is 1 or 99).ShouldNotBeEmpty();
|
||||
changes.Where(x => x.Handle is 1 or 99).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService && x.Value.WrappedValue.IsNull,
|
||||
"Good samples queued before demotion are first-delivered projected (D4)");
|
||||
changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not an ordinary tag");
|
||||
changes.ShouldContain(x => x.Handle == 3 && x.Value.StatusCode == StatusCodes.Good, "a condition child is not an ordinary tag");
|
||||
|
||||
var again = await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct);
|
||||
DataChanges(again.NotificationMessage).Where(x => x.Handle is 1 or 99)
|
||||
.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService, "Republish returns the recorded projected choice");
|
||||
}
|
||||
|
||||
/// <summary>Timing 2 (T0): messages prebuilt while Active (one notification per message), then demotion.</summary>
|
||||
[Fact]
|
||||
public async Task Timing2_prebuilt_messages_then_demotion_next_publish_is_projected_and_unsent_republish_is_unavailable()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717T2", "Warm", ct);
|
||||
var projection = rig.Server.CreateStandbyProjection()!;
|
||||
var raw = rig.EnsureRawVariable("p", "Double", writable: false);
|
||||
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
Promote(projection);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var client = (SessionClient)session;
|
||||
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 1, true, 0, ct)).SubscriptionId;
|
||||
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
|
||||
DataChanges((await PublishDataAsync(client, ct)).NotificationMessage).Count.ShouldBe(1);
|
||||
|
||||
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
|
||||
rig.Sink.WriteValue(raw, 20.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
rig.Sink.WriteValue(raw, 21.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
rig.Sink.WriteValue(raw, 22.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
|
||||
var a = await PublishDataAsync(client, ct); // builds several messages while Active; the first is delivered Good
|
||||
DataChanges(a.NotificationMessage).Single().Value.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
|
||||
DemoteDeliveryOnly(projection);
|
||||
var next = a.NotificationMessage.SequenceNumber + 1;
|
||||
(await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, next, ct).AsTask())).StatusCode
|
||||
.ShouldBe(StatusCodes.BadMessageNotAvailable, "2.0.1: a prebuilt, never-sent message cannot be first-delivered by Republish");
|
||||
var b = await PublishDataAsync(client, ct);
|
||||
b.NotificationMessage.SequenceNumber.ShouldBe(next);
|
||||
DataChanges(b.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService,
|
||||
"a message built while Active but first delivered after demotion is projected");
|
||||
var bRe = await client.RepublishAsync(H(), sub, next, ct);
|
||||
DataChanges(bRe.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
|
||||
}
|
||||
|
||||
/// <summary>Timing 3 (T0): Good delivered while Active, then demotion — Republish returns the original Good; after an
|
||||
/// acknowledge it is gone.</summary>
|
||||
[Fact]
|
||||
public async Task Timing3_good_delivered_while_active_republishes_good_after_demotion_and_ack_drops_it()
|
||||
{
|
||||
var ct = Deadline();
|
||||
await using var rig = await StartModeAsync("Warm717T3", "Warm", ct);
|
||||
var projection = rig.Server.CreateStandbyProjection()!;
|
||||
var raw = rig.EnsureRawVariable("d", "Double", writable: false);
|
||||
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
Promote(projection);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var client = (SessionClient)session;
|
||||
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
|
||||
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
|
||||
var first = await PublishDataAsync(client, ct);
|
||||
DataChanges(first.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
|
||||
|
||||
Demote(projection);
|
||||
var again = await client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct);
|
||||
DataChanges(again.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good,
|
||||
"Republish of a message first delivered while Active returns the original (D4 boundary)");
|
||||
|
||||
await client.PublishAsync(H(), new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = first.NotificationMessage.SequenceNumber } }, ct);
|
||||
(await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct).AsTask()))
|
||||
.StatusCode.ShouldBe(StatusCodes.BadMessageNotAvailable);
|
||||
}
|
||||
|
||||
/// <summary>The D4 boundary alone (spec §4.7 demotion step 2), without the projection that follows it, so the
|
||||
/// decision record is what the test observes.</summary>
|
||||
private static void DemoteDeliveryOnly(IStandbyProjection p) => p.SetStandbyDelivery(true);
|
||||
|
||||
// ------------------------------------------------------------------ raw client helpers
|
||||
|
||||
private static async Task<uint> AddItemAsync(SessionClient client, uint sub, NodeId node, uint handle, uint queue, CancellationToken ct,
|
||||
MonitoringFilter? filter = null)
|
||||
{
|
||||
var r = await client.CreateMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
|
||||
{
|
||||
new MonitoredItemCreateRequest
|
||||
{
|
||||
ItemToMonitor = new ReadValueId { NodeId = node, AttributeId = Attributes.Value },
|
||||
MonitoringMode = MonitoringMode.Reporting,
|
||||
RequestedParameters = new MonitoringParameters
|
||||
{
|
||||
ClientHandle = handle, SamplingInterval = 0, QueueSize = queue, DiscardOldest = true,
|
||||
Filter = filter is null ? ExtensionObject.Null : new ExtensionObject(filter),
|
||||
},
|
||||
},
|
||||
}, ct);
|
||||
StatusCode.IsGood(r.Results[0].StatusCode).ShouldBeTrue($"create {node}: {r.Results[0].StatusCode}");
|
||||
return r.Results[0].MonitoredItemId;
|
||||
}
|
||||
|
||||
internal static List<(uint Handle, DataValue Value)> DataChanges(NotificationMessage m)
|
||||
{
|
||||
var list = new List<(uint, DataValue)>();
|
||||
for (var i = 0; i < m.NotificationData.Count; i++)
|
||||
{
|
||||
if (m.NotificationData[i].TryGetValue<DataChangeNotification>(out var d))
|
||||
for (var j = 0; j < d.MonitoredItems.Count; j++) list.Add((d.MonitoredItems[j].ClientHandle, d.MonitoredItems[j].Value));
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static async Task<PublishResponse> PublishDataAsync(SessionClient client, CancellationToken ct)
|
||||
{
|
||||
for (var i = 0; i < 40; i++)
|
||||
{
|
||||
var r = await client.PublishAsync(H(), default, ct);
|
||||
if (r.NotificationMessage.NotificationData.Count > 0) return r;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("no data message");
|
||||
}
|
||||
|
||||
private static async Task<List<(uint Handle, DataValue Value)>> CollectAsync(SessionClient client, CancellationToken ct, TimeSpan window)
|
||||
{
|
||||
var collected = new List<(uint, DataValue)>();
|
||||
var until = DateTime.UtcNow + window;
|
||||
while (DateTime.UtcNow < until)
|
||||
{
|
||||
var r = await client.PublishAsync(H(), default, ct);
|
||||
collected.AddRange(DataChanges(r.NotificationMessage));
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
using Opc.Ua;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.6 (the T0-adopted mechanism) at unit level: the ordinary-tag marker and its validation, the
|
||||
/// Standby rewrite, the (SubscriptionId, SequenceNumber) decision record and its bounds, and the redundancy option
|
||||
/// rules of spec §3.
|
||||
/// </summary>
|
||||
public sealed class WarmStandbyUnitTests
|
||||
{
|
||||
private static DataValue Good(double v, int secondsAgo = 0) =>
|
||||
new(new Variant(v), StatusCodes.Good, (DateTimeUtc)DateTime.UtcNow.AddSeconds(-secondsAgo), DateTimeUtc.Now);
|
||||
|
||||
private static MonitoredItemNotification Marked(uint handle, DataValue value)
|
||||
{
|
||||
var n = new MonitoredItemNotification { ClientHandle = handle, Value = value };
|
||||
n.Message = new OrdinaryTagMarker(handle, value);
|
||||
return n;
|
||||
}
|
||||
|
||||
private static NotificationMessage Message(uint seq, params MonitoredItemNotification[] items) => new()
|
||||
{
|
||||
SequenceNumber = seq,
|
||||
PublishTime = DateTimeUtc.Now,
|
||||
NotificationData = new List<ExtensionObject>
|
||||
{
|
||||
new(new DataChangeNotification { MonitoredItems = items.ToList() }),
|
||||
},
|
||||
};
|
||||
|
||||
private static List<MonitoredItemNotification> Items(NotificationMessage m)
|
||||
{
|
||||
m.NotificationData[0].TryGetValue<DataChangeNotification>(out var d).ShouldBeTrue();
|
||||
return Enumerable.Range(0, d!.MonitoredItems.Count).Select(i => d.MonitoredItems[i]).ToList();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ marker + rewrite
|
||||
|
||||
[Fact]
|
||||
public void A_marked_ordinary_tag_notification_is_rewritten_and_the_input_is_not_mutated()
|
||||
{
|
||||
var value = Good(5);
|
||||
var tag = Marked(1, value);
|
||||
var other = new MonitoredItemNotification { ClientHandle = 2, Value = Good(6) };
|
||||
var input = Message(7, tag, other);
|
||||
|
||||
var projected = StandbyNotificationRewriter.Project(input, out var rewritten);
|
||||
|
||||
rewritten.ShouldBe(1);
|
||||
ReferenceEquals(projected, input).ShouldBeFalse();
|
||||
var items = Items(projected);
|
||||
items[0].ClientHandle.ShouldBe(1u);
|
||||
items[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService);
|
||||
items[0].Value.WrappedValue.IsNull.ShouldBeTrue();
|
||||
items[0].Value.ServerTimestamp.ShouldBe(value.ServerTimestamp);
|
||||
ReferenceEquals(items[1], other).ShouldBeTrue("a non-ordinary notification is passed through untouched");
|
||||
projected.SequenceNumber.ShouldBe(7u);
|
||||
|
||||
// The SDK-owned input (the per-delivery clone) is untouched.
|
||||
Items(input)[0].Value.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
ReferenceEquals(Items(input)[0], tag).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_stale_marker_from_a_recycled_payload_is_ignored()
|
||||
{
|
||||
// 2.0.1 pools MonitoredItemNotification payloads and Initialize() resets only ClientHandle and Value, so a
|
||||
// recycled object can carry a marker made for another notification. Handle or value mismatch ⇒ not ordinary.
|
||||
var otherHandle = new MonitoredItemNotification { ClientHandle = 3, Value = Good(5) };
|
||||
otherHandle.Message = new OrdinaryTagMarker(1, otherHandle.Value);
|
||||
var otherValue = new MonitoredItemNotification { ClientHandle = 1, Value = Good(6) };
|
||||
otherValue.Message = new OrdinaryTagMarker(1, Good(5, secondsAgo: 10));
|
||||
|
||||
OrdinaryTagMarker.IsOrdinaryTag(otherHandle).ShouldBeFalse();
|
||||
OrdinaryTagMarker.IsOrdinaryTag(otherValue).ShouldBeFalse();
|
||||
var input = Message(1, otherHandle, otherValue);
|
||||
ReferenceEquals(StandbyNotificationRewriter.Project(input, out var rewritten), input).ShouldBeTrue();
|
||||
rewritten.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Bad_and_unmarked_notifications_are_never_rewritten()
|
||||
{
|
||||
var bad = Marked(1, new DataValue(Variant.Null, StatusCodes.BadNoCommunication, DateTimeUtc.Now, DateTimeUtc.Now));
|
||||
var unmarked = new MonitoredItemNotification { ClientHandle = 2, Value = Good(1) };
|
||||
var input = Message(1, bad, unmarked);
|
||||
ReferenceEquals(StandbyNotificationRewriter.Project(input, out _), input).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uncertain_ordinary_tag_values_are_projected_too()
|
||||
{
|
||||
var uncertain = Marked(1, new DataValue(new Variant(1.0), StatusCodes.UncertainLastUsableValue, DateTimeUtc.Now, DateTimeUtc.Now));
|
||||
Items(StandbyNotificationRewriter.Project(Message(1, uncertain), out _))[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ SDK-field contract (regression pin)
|
||||
|
||||
/// <summary>
|
||||
/// The mechanism stands on two SDK 2.0.1 behaviours of <see cref="MonitoredItemNotification.Message"/>: the SDK's
|
||||
/// per-delivery <c>CoreUtils.Clone</c> COPIES it, and the binary encoder NEVER writes it. An SDK upgrade that
|
||||
/// changes either breaks D4 silently (no marker ⇒ no rewrite) or leaks a server object onto the wire — this
|
||||
/// fails loudly instead.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Sdk_contract_the_marker_survives_the_delivery_clone_and_is_never_encoded()
|
||||
{
|
||||
var value = Good(5);
|
||||
var marked = Message(9, Marked(1, value));
|
||||
|
||||
var clone = CoreUtils.Clone(marked);
|
||||
ReferenceEquals(clone, marked).ShouldBeFalse();
|
||||
var cloned = Items(clone!)[0];
|
||||
ReferenceEquals(cloned, Items(marked)[0]).ShouldBeFalse("the clone is deep");
|
||||
cloned.Message.ShouldBeOfType<OrdinaryTagMarker>("the SDK clone must copy MonitoredItemNotification.Message");
|
||||
OrdinaryTagMarker.IsOrdinaryTag(cloned).ShouldBeTrue("the copied marker still validates against the clone");
|
||||
|
||||
var context = ServiceMessageContext.Create(DefaultTelemetry.Create(_ => { }));
|
||||
var unmarked = Message(9, new MonitoredItemNotification { ClientHandle = 1, Value = value });
|
||||
unmarked.PublishTime = marked.PublishTime;
|
||||
var withMarker = BinaryEncoder.EncodeMessage(marked, context);
|
||||
var without = BinaryEncoder.EncodeMessage(unmarked, context);
|
||||
withMarker.ShouldBe(without, "the helper field must never reach the wire");
|
||||
var decoded = BinaryDecoder.DecodeMessage<NotificationMessage>(withMarker, context);
|
||||
Items(decoded!)[0].Message.ShouldBeNull();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ decision record
|
||||
|
||||
[Fact]
|
||||
public void The_first_delivery_records_the_decision_and_every_later_delivery_reuses_it()
|
||||
{
|
||||
var record = new FirstDeliveryRecord(standby: false);
|
||||
record.Decide(1, 10).ShouldBeFalse();
|
||||
record.SetStandby(true);
|
||||
record.Decide(1, 10).ShouldBeFalse("a Republish of a message first delivered Active stays Active (D4)");
|
||||
record.Decide(1, 11).ShouldBeTrue("a first delivery after the flip is Standby");
|
||||
record.SetStandby(false);
|
||||
record.Decide(1, 11).ShouldBeTrue("a Republish of a message first delivered Standby stays projected");
|
||||
record.Decide(2, 11).ShouldBeFalse("keyed by subscription too");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_record_is_pruned_by_available_sequence_numbers_and_bounded()
|
||||
{
|
||||
var record = new FirstDeliveryRecord(standby: true);
|
||||
for (uint s = 1; s <= 5; s++) record.Decide(1, s);
|
||||
record.Prune(1, [3, 4], justDelivered: 5);
|
||||
record.Count.ShouldBe(3);
|
||||
|
||||
for (uint s = 100; s < 100 + FirstDeliveryRecord.MaxPerSubscription + 50; s++) record.Decide(1, s);
|
||||
record.Count.ShouldBe(FirstDeliveryRecord.MaxPerSubscription);
|
||||
|
||||
record.Decide(2, 1);
|
||||
record.SweepDead(new HashSet<uint> { 2 });
|
||||
record.SubscriptionCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dead_subscriptions_are_evicted_least_recently_used_beyond_the_bound()
|
||||
{
|
||||
var record = new FirstDeliveryRecord(standby: false);
|
||||
for (uint sub = 1; sub <= FirstDeliveryRecord.MaxSubscriptions; sub++) record.Decide(sub, 1);
|
||||
record.Decide(1, 2); // touch subscription 1 so 2 is the least recently used
|
||||
record.Decide(FirstDeliveryRecord.MaxSubscriptions + 1, 1);
|
||||
record.SubscriptionCount.ShouldBe(FirstDeliveryRecord.MaxSubscriptions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A role flip and a first delivery race (spec §4.6 item 2): the flip waits for a decision in progress, so the
|
||||
/// decision is ordered strictly before or after it — never torn. Driven by a deterministic barrier, not sleeps.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_flip_waits_for_a_decision_in_progress_and_orders_strictly_against_it()
|
||||
{
|
||||
for (var round = 0; round < 50; round++)
|
||||
{
|
||||
var record = new FirstDeliveryRecord(standby: false);
|
||||
using var inside = new ManualResetEventSlim();
|
||||
using var release = new ManualResetEventSlim();
|
||||
record.DecisionBarrier = () => { inside.Set(); release.Wait(TimeSpan.FromSeconds(10)); };
|
||||
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var decision = Task.Run(() => record.Decide(1, (uint)round + 1), ct);
|
||||
inside.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue();
|
||||
var flip = Task.Run(() => record.SetStandby(true), ct);
|
||||
(await Task.WhenAny(flip, Task.Delay(100, ct))).ShouldNotBe(flip, "the flip must wait for the decision in progress");
|
||||
release.Set();
|
||||
(await decision).ShouldBeFalse("the decision read the state before the flip");
|
||||
await flip;
|
||||
record.DecisionBarrier = null;
|
||||
record.Decide(1, (uint)round + 1).ShouldBeFalse("and recorded it");
|
||||
record.Decide(1, (uint)round + 1000).ShouldBeTrue("a later first delivery sees the flip");
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ WarmStandbyState
|
||||
|
||||
[Fact]
|
||||
public void The_Off_state_never_projects_suppresses_or_refuses()
|
||||
{
|
||||
var off = new WarmStandbyState(false);
|
||||
off.DataProjected = true;
|
||||
off.AlarmsSuppressed = true;
|
||||
off.DataProjected.ShouldBeFalse();
|
||||
off.AlarmsSuppressed.ShouldBeFalse();
|
||||
off.RefusesMutations.ShouldBeFalse();
|
||||
off.Delivery.ShouldBeNull();
|
||||
|
||||
var warm = new WarmStandbyState(true);
|
||||
warm.DataProjected.ShouldBeTrue("a Warm node boots Standby");
|
||||
warm.AlarmsSuppressed.ShouldBeTrue();
|
||||
warm.RefusesMutations.ShouldBeTrue();
|
||||
warm.Delivery!.Standby.ShouldBeTrue();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ OpcUa:Redundancy rules (spec §3)
|
||||
|
||||
[Theory]
|
||||
[InlineData("Unset", new string[0], true)]
|
||||
[InlineData("", new string[0], true)]
|
||||
[InlineData("none", new string[0], true)]
|
||||
[InlineData("None", new[] { "urn:peer" }, false)]
|
||||
[InlineData("Warm", new[] { "urn:peer" }, true)]
|
||||
[InlineData("WARM", new[] { "urn:peer", "urn:peer2" }, true)]
|
||||
[InlineData("Warm", new string[0], false)]
|
||||
[InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)]
|
||||
[InlineData("Warm", new[] { "urn:self" }, false)]
|
||||
[InlineData("Hot", new[] { "urn:peer" }, false)]
|
||||
[InlineData("2", new[] { "urn:peer" }, false)]
|
||||
[InlineData("Transparent", new[] { "urn:peer" }, false)]
|
||||
public void Redundancy_option_rules(string mode, string[] peers, bool valid)
|
||||
{
|
||||
var options = new OpcUaRedundancyOptions { Mode = mode };
|
||||
options.Check("urn:self", peers).Count.ShouldBe(valid ? 0 : 1, string.Join("; ", options.Check("urn:self", peers)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Redundancy_defaults_are_Unset_with_promote_on()
|
||||
{
|
||||
var options = new OpcUaApplicationHostOptions();
|
||||
options.Redundancy.ParsedMode.ShouldBe(OpcUaRedundancyMode.Unset);
|
||||
options.Redundancy.PromoteWhilePrimaryUnreachable.ShouldBeTrue();
|
||||
options.Redundancy.ToSettings().ShouldBe(WarmRedundancySettings.Unset);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.1–§4.3 and §4.7 in <see cref="OpcUaPublishActor"/>: the Warm verdict from membership + the
|
||||
/// accepted revisioned snapshot, the bands, and the transition step order — with a recording projection and
|
||||
/// ServiceLevel publisher writing into ONE ordered log, so the invariant "an Active-band level is never visible over
|
||||
/// projected data" is checked on the actual sequence.
|
||||
/// </summary>
|
||||
public sealed class OpcUaPublishActorWarmTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
|
||||
|
||||
private readonly ConcurrentQueue<string> _log = new();
|
||||
private long _seq;
|
||||
|
||||
private NodeId Self
|
||||
{
|
||||
get
|
||||
{
|
||||
var cluster = Akka.Cluster.Cluster.Get(Sys);
|
||||
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
|
||||
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
|
||||
}
|
||||
}
|
||||
|
||||
private IActorRef Start(WarmRedundancySettings redundancy, LoggingPublisher? publisher = null) =>
|
||||
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher ?? new LoggingPublisher(_log), localNode: Self, dbLess: true,
|
||||
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: redundancy,
|
||||
standbyProjection: new RecordingProjection(_log)));
|
||||
|
||||
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
|
||||
new(new[]
|
||||
{
|
||||
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
|
||||
false, primary == Self, DateTime.UtcNow),
|
||||
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
|
||||
false, primary == Other, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId(),
|
||||
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
|
||||
|
||||
private string[] Steps => _log.ToArray();
|
||||
|
||||
private void AwaitLogEndsWith(params string[] tail) =>
|
||||
AwaitAssert(() => Steps.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", Steps)), PresenceBudget,
|
||||
cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
private static readonly string[] Promotion = ["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL:250"];
|
||||
private static readonly string[] Demotion = ["SL:90", "delivery:True", "alarms:True", "proj:True", "queue:True"];
|
||||
|
||||
[Fact]
|
||||
public void Promotion_and_demotion_run_in_the_spec_order()
|
||||
{
|
||||
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
||||
actor.Tell(Snapshot(Self));
|
||||
AwaitLogEndsWith(Promotion);
|
||||
Steps.ShouldBe(Promotion, "boot is Standby; the first accepted snapshot promotes and nothing else ran");
|
||||
|
||||
actor.Tell(Snapshot(Other));
|
||||
AwaitLogEndsWith(Demotion);
|
||||
AssertNoActiveLevelOverProjectedData();
|
||||
}
|
||||
|
||||
/// <summary>Spec §4.7 invariant over the whole sequence: every published level ≥ 100 comes after the projection was
|
||||
/// last opened, never while it is closed.</summary>
|
||||
private void AssertNoActiveLevelOverProjectedData()
|
||||
{
|
||||
var projected = true; // a Warm node boots projected
|
||||
foreach (var step in Steps)
|
||||
{
|
||||
if (step == "proj:True") projected = true;
|
||||
else if (step == "proj:False") projected = false;
|
||||
else if (step.StartsWith("SL:", StringComparison.Ordinal) && byte.Parse(step[3..]) >= 100)
|
||||
projected.ShouldBeFalse($"an Active-band level was visible over projected data: {string.Join(" ", Steps)}");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unrevisioned_snapshot_never_makes_a_Warm_node_Active()
|
||||
{
|
||||
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
||||
actor.Tell(Snapshot(Self, revisioned: false));
|
||||
AwaitLogEndsWith("SL:90");
|
||||
Steps.ShouldBe(new[] { "SL:90" });
|
||||
}
|
||||
|
||||
/// <summary>Spec §4.2: after a newer snapshot names this node Primary, an OLDER incarnation's late publish naming the
|
||||
/// peer is dropped — the node stays Active. Then a genuinely newer one demotes it (positive control).</summary>
|
||||
[Fact]
|
||||
public void An_older_revision_after_a_newer_one_is_dropped()
|
||||
{
|
||||
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
||||
actor.Tell(Snapshot(Self, upNumber: 5));
|
||||
AwaitLogEndsWith(Promotion);
|
||||
|
||||
actor.Tell(Snapshot(Other, upNumber: 4)); // a departed older singleton's late heartbeat
|
||||
actor.Tell(Snapshot(Self, upNumber: 5)); // a fence: processed after the stale one
|
||||
AwaitAssert(() => Steps.Count(s => s == "SL:250").ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
|
||||
Steps.ShouldNotContain("delivery:True", "the stale snapshot must not demote");
|
||||
|
||||
actor.Tell(Snapshot(Other, upNumber: 6));
|
||||
AwaitLogEndsWith(Demotion);
|
||||
}
|
||||
|
||||
/// <summary>Spec §4.7: if the Active level cannot be published, the promotion stops before it and retries; the
|
||||
/// Standby level stays visible meanwhile.</summary>
|
||||
[Fact]
|
||||
public void A_failed_Active_level_publish_halts_before_it_and_retries()
|
||||
{
|
||||
var publisher = new LoggingPublisher(_log) { FailNext = 1, FailLevel = 250 };
|
||||
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
|
||||
actor.Tell(Snapshot(Self));
|
||||
AwaitAssert(() => Steps.ShouldContain("SL:250"), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
var log = Steps;
|
||||
log.Take(6).ShouldBe(["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL-FAILED:250"]);
|
||||
log.Last().ShouldBe("SL:250", "retried without a new snapshot");
|
||||
}
|
||||
|
||||
/// <summary>A demotion continues when the Standby level publish fails: failing safe degrades the data.</summary>
|
||||
[Fact]
|
||||
public void A_failed_Standby_level_publish_still_degrades_the_data()
|
||||
{
|
||||
var publisher = new LoggingPublisher(_log);
|
||||
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher);
|
||||
actor.Tell(Snapshot(Self));
|
||||
AwaitLogEndsWith(Promotion);
|
||||
publisher.FailNext = 1;
|
||||
publisher.FailLevel = 90;
|
||||
actor.Tell(Snapshot(Other));
|
||||
AwaitLogEndsWith("SL-FAILED:90", "delivery:True", "alarms:True", "proj:True", "queue:True");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_burst_of_alternating_verdicts_converges_on_the_last()
|
||||
{
|
||||
var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm));
|
||||
for (var i = 0; i < 20; i++) actor.Tell(Snapshot(i % 2 == 0 ? Self : Other));
|
||||
actor.Tell(Snapshot(Self));
|
||||
AwaitLogEndsWith(Promotion);
|
||||
AssertNoActiveLevelOverProjectedData();
|
||||
}
|
||||
|
||||
/// <summary>The default-off pin at the actor: Unset never touches the projection and keeps today's levels.</summary>
|
||||
[Fact]
|
||||
public void Unset_never_drives_the_projection_and_keeps_todays_levels()
|
||||
{
|
||||
var actor = Start(WarmRedundancySettings.Unset);
|
||||
actor.Tell(Snapshot(Self));
|
||||
AwaitLogEndsWith("SL:250");
|
||||
actor.Tell(Snapshot(Other));
|
||||
AwaitLogEndsWith("SL:240");
|
||||
Steps.ShouldBe(new[] { "SL:250", "SL:240" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_deployed_cluster_mode_that_disagrees_with_the_local_mode_is_logged_and_counted()
|
||||
{
|
||||
var artifact = Encoding.UTF8.GetBytes(
|
||||
"{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":\"None\"}],\"Nodes\":[]}");
|
||||
DeploymentArtifact.ReadClusterRedundancyMode(artifact, "x:1").ShouldBe("None");
|
||||
DeploymentArtifact.ReadClusterRedundancyMode(
|
||||
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":1}]}"), "x:1").ShouldBe("Warm");
|
||||
DeploymentArtifact.ReadClusterRedundancyMode(
|
||||
Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"a\",\"RedundancyMode\":\"None\"},{\"ClusterId\":\"b\",\"RedundancyMode\":\"Hot\"}],"
|
||||
+ "\"Nodes\":[{\"NodeId\":\"x:1\",\"ClusterId\":\"b\"}]}"), "x:1").ShouldBe("Hot");
|
||||
|
||||
using var listener = new System.Diagnostics.Metrics.MeterListener();
|
||||
long mismatches = 0;
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (ReferenceEquals(instrument, ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.RedundancyModeMismatch))
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((_, value, _, _) => Interlocked.Add(ref mismatches, value));
|
||||
listener.Start();
|
||||
|
||||
var warmActor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
localNode: NodeId.Parse("x:1"), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm)));
|
||||
EventFilter.Error(contains: "RedundancyMode for this node's cluster is None").ExpectOne(() =>
|
||||
warmActor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), Artifact: artifact)),
|
||||
TestContext.Current.CancellationToken);
|
||||
Interlocked.Read(ref mismatches).ShouldBeGreaterThanOrEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Writes each published level into the shared log; can be told to fail the next N publishes of a level.</summary>
|
||||
internal sealed class LoggingPublisher(ConcurrentQueue<string> log) : IServiceLevelPublisher
|
||||
{
|
||||
public int FailNext { get; set; }
|
||||
public byte FailLevel { get; set; }
|
||||
|
||||
public void Publish(byte serviceLevel)
|
||||
{
|
||||
if (FailNext > 0 && serviceLevel == FailLevel)
|
||||
{
|
||||
FailNext--;
|
||||
log.Enqueue($"SL-FAILED:{serviceLevel}");
|
||||
throw new InvalidOperationException("injected ServiceLevel publish failure");
|
||||
}
|
||||
|
||||
log.Enqueue($"SL:{serviceLevel}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
|
||||
|
||||
/// <summary>Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7).</summary>
|
||||
public sealed class WarmRoleAuthorityTests
|
||||
{
|
||||
private static readonly NodeId Self = NodeId.Parse("a:1");
|
||||
private static readonly NodeId Peer = NodeId.Parse("b:1");
|
||||
|
||||
private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true,
|
||||
NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) =>
|
||||
new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote);
|
||||
|
||||
[Fact]
|
||||
public void Active_iff_membership_and_the_accepted_snapshot_both_name_self()
|
||||
{
|
||||
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active);
|
||||
WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf)
|
||||
{
|
||||
var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self));
|
||||
verdict.Role.ShouldBe(WarmRole.Standby);
|
||||
verdict.Disagreement.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership()
|
||||
{
|
||||
var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null));
|
||||
verdict.Role.ShouldBe(WarmRole.Standby);
|
||||
verdict.Reason.ShouldBe("boot");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Not_up_or_endpoint_failed_is_Standby()
|
||||
{
|
||||
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby);
|
||||
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby);
|
||||
}
|
||||
|
||||
/// <summary>Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes.</summary>
|
||||
[Fact]
|
||||
public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option()
|
||||
{
|
||||
var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true);
|
||||
WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active);
|
||||
WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
|
||||
WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby,
|
||||
"a reachable Primary is never overridden");
|
||||
WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role
|
||||
.ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_late_bind_replays_the_retained_state_but_not_the_actions()
|
||||
{
|
||||
var deferred = new DeferredStandbyProjection();
|
||||
deferred.SetDataProjection(false);
|
||||
deferred.SetStandbyDelivery(false);
|
||||
deferred.SetAlarmSuppression(false);
|
||||
deferred.QueueCurrentToLiveItems(false).ShouldBe(0);
|
||||
deferred.RefreshAllEventItems().ShouldBe(0);
|
||||
|
||||
var inner = new RecordingProjection();
|
||||
deferred.SetInner(inner);
|
||||
inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" });
|
||||
deferred.QueueCurrentToLiveItems(true);
|
||||
inner.Log.Last().ShouldBe("queue:True");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records every <see cref="IStandbyProjection"/> step into a shared, ordered log.</summary>
|
||||
internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue<string>? log = null) : IStandbyProjection
|
||||
{
|
||||
public System.Collections.Concurrent.ConcurrentQueue<string> Log { get; } = log ?? new();
|
||||
public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}");
|
||||
public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}");
|
||||
public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}");
|
||||
public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; }
|
||||
public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; }
|
||||
}
|
||||
Reference in New Issue
Block a user