feat(717): Warm non-transparent redundancy, default-off (T1-T10)

OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start).
- T1 options + validator + effective-URI check + boot banner; Warm needs the driver role.
- T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only
  ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None
  and the SDK's orphan i=11314 deleted; Unset untouched.
- T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older
  revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active.
- T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable.
- T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec
  4.7 order (Active level last and retried; Standby level first, data degraded regardless).
- T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState.
- T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by
  (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's
  cached copy is never mutated.
- T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters.
- T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion
  refresh via ISubscription.ConditionRefresh2Async on every live event item.
- T10 deploy-time RedundancyMode cross-check + six instruments.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:26:36 -05:00
parent dcff5ae994
commit 7829b937d1
27 changed files with 3056 additions and 29 deletions
@@ -62,4 +62,44 @@ public static class ServiceLevelCalculator
return (byte)Math.Clamp(basis + (h.IsDriverPrimary ? 10 : 0), 0, 255);
}
/// <summary>Warm Standby, healthy (Gitea #717 spec §4.3): Degraded band.</summary>
public const byte WarmStandbyHealthy = 90;
/// <summary>Warm Standby, stale or DB unreachable (spec §4.3): Degraded band.</summary>
public const byte WarmStandbyDegraded = 50;
/// <summary>Warm Active, critically degraded (spec §4.3): the lowest level a data-serving Active node publishes.</summary>
public const byte WarmActiveCritical = 100;
/// <summary>Warm Active, stale config with the DB reachable (spec §4.3).</summary>
public const byte WarmActiveStale = 200;
/// <summary>Warm Active, healthy (spec §4.3): 240 + the 10-point Primary bonus.</summary>
public const byte WarmActiveHealthy = 250;
/// <summary>
/// The Warm ServiceLevel bands (Gitea #717 spec §4.3). <see cref="Compute"/> is unchanged and still serves
/// <c>Unset</c> and <c>None</c>; this is used only under <c>OpcUa:Redundancy:Mode=Warm</c>.
/// </summary>
/// <remarks>
/// Invariant: every Standby level (≤ <see cref="WarmStandbyHealthy"/>) is below every level a data-serving
/// Active node publishes (≥ <see cref="WarmActiveCritical"/>), so a client that picks the highest
/// ServiceLevel never prefers a Standby. A member that is not Up/Joining, or whose own endpoint failed, is 0
/// whatever its role (unchanged from <see cref="Compute"/>, lmxopcua#762).
/// </remarks>
/// <param name="h">The node's health facts; <see cref="NodeHealthInputs.IsDriverPrimary"/> is ignored — the
/// Warm verdict <paramref name="isActive"/> replaces it.</param>
/// <param name="isActive">The Warm role verdict (spec §4.1).</param>
/// <returns>The ServiceLevel byte.</returns>
public static byte ComputeWarm(NodeHealthInputs h, bool isActive)
{
if (h.MemberState is not (MemberStatus.Up or MemberStatus.Joining)) return 0;
if (!h.EndpointListening) return 0;
var healthy = h.DbReachable && h.OpcUaProbeOk && !h.Stale;
if (!isActive) return healthy ? WarmStandbyHealthy : WarmStandbyDegraded;
if (healthy) return WarmActiveHealthy;
return h.DbReachable && h.Stale ? WarmActiveStale : WarmActiveCritical;
}
}
@@ -6,6 +6,74 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
/// Broadcast whenever the cluster's redundancy topology changes (node up/down, role-leader
/// change, partition heal). Subscribers compute their local OPC UA ServiceLevel from this.
/// </summary>
/// <param name="Nodes">One entry per cluster member.</param>
/// <param name="CorrelationId">A random id per publish (diagnostics only; it carries no order).</param>
/// <param name="Revision">The publisher's ordered revision (Gitea #717, spec §4.2). <c>null</c> from a pre-#717
/// publisher during a rolling upgrade: such a snapshot still feeds the existing gates, but can never make a Warm
/// node Active.</param>
public sealed record RedundancyStateChanged(
IReadOnlyList<NodeRedundancyState> Nodes,
CorrelationId CorrelationId);
CorrelationId CorrelationId,
SnapshotRevision? Revision = null);
/// <summary>
/// The order of <see cref="RedundancyStateChanged"/> snapshots (Gitea #717, spec §4.2), compared
/// lexicographically. Within one cluster incarnation the singleton always lives on the oldest member, and every
/// surviving member has a larger up-number than a departed oldest one, so <see cref="PublisherUpNumber"/> is
/// monotonic across handovers; <see cref="IncarnationTick"/> orders a singleton restarted in the same process;
/// <see cref="Sequence"/> orders the publishes of one incarnation.
/// </summary>
/// <param name="PublisherUpNumber">The <c>Member.UpNumber</c> of the node hosting the singleton.</param>
/// <param name="IncarnationTick"><c>Environment.TickCount64</c> at the singleton's <c>PreStart</c>.</param>
/// <param name="Sequence">Incremented on every publish, heartbeats included.</param>
public sealed record SnapshotRevision(long PublisherUpNumber, long IncarnationTick, long Sequence)
: IComparable<SnapshotRevision>
{
/// <inheritdoc />
public int CompareTo(SnapshotRevision? other)
{
if (other is null) return 1;
var c = PublisherUpNumber.CompareTo(other.PublisherUpNumber);
if (c != 0) return c;
c = IncarnationTick.CompareTo(other.IncarnationTick);
return c != 0 ? c : Sequence.CompareTo(other.Sequence);
}
/// <inheritdoc />
public override string ToString() => $"{PublisherUpNumber}.{IncarnationTick}.{Sequence}";
}
/// <summary>Why a snapshot was not accepted by a <see cref="SnapshotRevisionGate"/>.</summary>
public enum SnapshotRejection
{
/// <summary>Accepted.</summary>
None,
/// <summary>Its revision is not greater than the last accepted one (a late publish of an older incarnation).</summary>
OlderRevision,
/// <summary>It carries no revision, while revisioned snapshots are being ordered.</summary>
Unrevisioned,
}
/// <summary>
/// The subscriber-side order rule of spec §4.2: accept a snapshot only if its revision is greater than the last
/// accepted one. Not thread-safe; owned by one actor.
/// </summary>
public sealed class SnapshotRevisionGate
{
/// <summary>The last accepted revision; <c>null</c> until a revisioned snapshot was accepted.</summary>
public SnapshotRevision? LastAccepted { get; private set; }
/// <summary>Decide one snapshot. An unrevisioned snapshot is reported <see cref="SnapshotRejection.Unrevisioned"/>
/// and never advances the order (the caller decides whether the existing gates still use it).</summary>
/// <param name="msg">The snapshot.</param>
/// <returns><see cref="SnapshotRejection.None"/> when accepted.</returns>
public SnapshotRejection Offer(RedundancyStateChanged msg)
{
if (msg.Revision is not { } revision) return SnapshotRejection.Unrevisioned;
if (LastAccepted is { } last && revision.CompareTo(last) <= 0) return SnapshotRejection.OlderRevision;
LastAccepted = revision;
return SnapshotRejection.None;
}
}
@@ -190,6 +190,44 @@ public static class OtOpcUaTelemetry
Meter.CreateCounter<long>("otopcua.opcua.condition_events.superseded", unit: "{delivery}",
description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state.");
// ---------------- Warm redundancy (Gitea #717, spec §7) ----------------
private static int _warmRole = -1;
/// <summary>Records this process's Warm role for <see cref="WarmRole"/>: 1 Active, 0 Standby, −1 not Warm.</summary>
/// <param name="value">The role value.</param>
public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value);
/// <summary>The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm.</summary>
public static readonly ObservableGauge<int> WarmRole =
Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}",
description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm.");
/// <summary>Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed).</summary>
public static readonly Counter<long> WarmTransition =
Meter.CreateCounter<long>("otopcua.redundancy.warm_transition", unit: "{transition}",
description: "Warm role transitions (direction=promote|demote, reason).");
/// <summary>Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned).</summary>
public static readonly Counter<long> SnapshotRejected =
Meter.CreateCounter<long>("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}",
description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned).");
/// <summary>The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.</summary>
public static readonly Counter<long> RedundancyModeMismatch =
Meter.CreateCounter<long>("otopcua.redundancy.mode_mismatch", unit: "{apply}",
description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.");
/// <summary>Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4).</summary>
public static readonly Counter<long> StandbyProjectedNotifications =
Meter.CreateCounter<long>("otopcua.opcua.standby.projected_notifications", unit: "{notification}",
description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery.");
/// <summary>New alarm condition events not delivered on a Warm Standby (still stamped).</summary>
public static readonly Counter<long> StandbySuppressedAlarmEvents =
Meter.CreateCounter<long>("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}",
description: "New alarm condition events suppressed on a Warm standby.");
// ---------------- Convenience helpers ----------------
/// <summary>
@@ -0,0 +1,112 @@
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
/// <summary>
/// The data-plane side of a Warm role transition (Gitea #717, spec §4.7). <c>OpcUaPublishActor</c> owns the
/// verdict and drives these steps in a fixed order; the node manager implements them. Every member is
/// non-blocking and idempotent. Under <see cref="OpcUaRedundancyMode.Unset"/> and
/// <see cref="OpcUaRedundancyMode.None"/> nothing calls it.
/// </summary>
public interface IStandbyProjection
{
/// <summary>Opens (<c>false</c>) or closes (<c>true</c>) the per-tag read projection: while projected, every
/// ordinary-tag <c>Value</c> read and sample answers <c>BadOutOfService</c>/null, and writes and Part 9 condition
/// methods are refused <c>BadInvalidState</c>. The stored value keeps updating from acquisition.</summary>
/// <param name="projected">Whether ordinary-tag values are projected.</param>
void SetDataProjection(bool projected);
/// <summary>Switches the first-delivery decision (spec §4.6, D4): a Publish response delivered for the first time
/// while <paramref name="standby"/> is recorded as Standby and its ordinary-tag notifications are rewritten.</summary>
/// <param name="standby">Whether first deliveries are decided Standby.</param>
void SetStandbyDelivery(bool standby);
/// <summary>Suppresses (<c>true</c>) or restores delivery of NEW alarm condition events (spec §4.5). Condition
/// state keeps being stamped, so a later refresh carries current, ackable state.</summary>
/// <param name="suppressed">Whether new alarm events are suppressed.</param>
void SetAlarmSuppression(bool suppressed);
/// <summary>Queues each live, non-stood-down ordinary-tag data item one value, ignoring its filters: the current
/// stored value (promotion) or <c>BadOutOfService</c>/null (demotion).</summary>
/// <param name="projected">Whether to queue the projected value instead of the stored one.</param>
/// <returns>The number of items queued to.</returns>
int QueueCurrentToLiveItems(bool projected);
/// <summary>Runs a ConditionRefresh on every live event item (Server-object, folder and direct), through the
/// existing refresh paths and their ACL decisions (spec §4.5 promotion refresh).</summary>
/// <returns>The number of event items refreshed.</returns>
int RefreshAllEventItems();
}
/// <summary>
/// Late-binding <see cref="IStandbyProjection"/> (the <see cref="DeferredServiceLevelPublisher"/> pattern): the
/// actor gets it at construction, the node manager only exists after SDK start. The latest requested
/// projection / delivery / suppression state is retained and replayed on bind (the #719 lesson); the two
/// queue/refresh actions are not replayed — before bind there are no items to act on.
/// </summary>
public sealed class DeferredStandbyProjection : IStandbyProjection
{
private readonly object _gate = new();
private IStandbyProjection? _inner;
private bool? _projected;
private bool? _standby;
private bool? _suppressed;
/// <summary>Bind (and replay the retained state), or unbind with <c>null</c>.</summary>
/// <param name="inner">The live implementation, or null to unbind.</param>
public void SetInner(IStandbyProjection? inner)
{
lock (_gate)
{
_inner = inner;
if (inner is null) return;
if (_projected is { } p) inner.SetDataProjection(p);
if (_standby is { } s) inner.SetStandbyDelivery(s);
if (_suppressed is { } a) inner.SetAlarmSuppression(a);
}
}
/// <inheritdoc />
public void SetDataProjection(bool projected)
{
lock (_gate)
{
_projected = projected;
_inner?.SetDataProjection(projected);
}
}
/// <inheritdoc />
public void SetStandbyDelivery(bool standby)
{
lock (_gate)
{
_standby = standby;
_inner?.SetStandbyDelivery(standby);
}
}
/// <inheritdoc />
public void SetAlarmSuppression(bool suppressed)
{
lock (_gate)
{
_suppressed = suppressed;
_inner?.SetAlarmSuppression(suppressed);
}
}
/// <inheritdoc />
public int QueueCurrentToLiveItems(bool projected)
{
IStandbyProjection? inner;
lock (_gate) inner = _inner;
return inner?.QueueCurrentToLiveItems(projected) ?? 0;
}
/// <inheritdoc />
public int RefreshAllEventItems()
{
IStandbyProjection? inner;
lock (_gate) inner = _inner;
return inner?.RefreshAllEventItems() ?? 0;
}
}
@@ -0,0 +1,36 @@
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
/// <summary>
/// The parsed <c>OpcUa:Redundancy:Mode</c> (Gitea #717). Startup-static: changing it requires a restart.
/// </summary>
public enum OpcUaRedundancyMode
{
/// <summary>Not opted in (the default): byte-for-byte the pre-#717 server. Nothing in the redundancy metadata,
/// ServiceLevel bands, projection or alarm delivery changes.</summary>
Unset = 0,
/// <summary>Standalone: advertises <c>RedundancySupport=None</c> with no <c>ServerUriArray</c>; never projects.</summary>
None = 1,
/// <summary>A 2-node pair with a degraded standby (spec §4): non-transparent Warm metadata, Standby bands below
/// every Active level, ordinary-tag values projected <c>BadOutOfService</c> on the Standby, new alarm events
/// suppressed there, and writes / Part 9 methods refused <c>BadInvalidState</c> there.</summary>
Warm = 2,
}
/// <summary>
/// The runtime half of <c>OpcUa:Redundancy</c> (Gitea #717): what the redundancy actors need. The Host builds it
/// from the validated <c>OpcUaApplicationHostOptions</c> and registers it as a singleton; a node that registers
/// none (a harness, an admin-only node) is <see cref="Unset"/>.
/// </summary>
/// <param name="Mode">The redundancy mode.</param>
/// <param name="PromoteWhilePrimaryUnreachable">Warm only: let the survivor go Active while the Primary is
/// unreachable but not yet downed (spec §4.1, owner answer Q2). Default on.</param>
public sealed record WarmRedundancySettings(OpcUaRedundancyMode Mode, bool PromoteWhilePrimaryUnreachable = true)
{
/// <summary>The not-opted-in default.</summary>
public static WarmRedundancySettings Unset { get; } = new(OpcUaRedundancyMode.Unset);
/// <summary>True when this node runs the Warm role contract.</summary>
public bool IsWarm => Mode == OpcUaRedundancyMode.Warm;
}