feat(717): Warm non-transparent redundancy, default-off (T1-T10)
OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start). - T1 options + validator + effective-URI check + boot banner; Warm needs the driver role. - T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None and the SDK's orphan i=11314 deleted; Unset untouched. - T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active. - T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable. - T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec 4.7 order (Active level last and retried; Standby level first, data degraded regardless). - T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState. - T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's cached copy is never mutated. - T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters. - T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion refresh via ISubscription.ConditionRefresh2Async on every live event item. - T10 deploy-time RedundancyMode cross-check + six instruments. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -62,4 +62,44 @@ public static class ServiceLevelCalculator
|
||||
|
||||
return (byte)Math.Clamp(basis + (h.IsDriverPrimary ? 10 : 0), 0, 255);
|
||||
}
|
||||
|
||||
/// <summary>Warm Standby, healthy (Gitea #717 spec §4.3): Degraded band.</summary>
|
||||
public const byte WarmStandbyHealthy = 90;
|
||||
|
||||
/// <summary>Warm Standby, stale or DB unreachable (spec §4.3): Degraded band.</summary>
|
||||
public const byte WarmStandbyDegraded = 50;
|
||||
|
||||
/// <summary>Warm Active, critically degraded (spec §4.3): the lowest level a data-serving Active node publishes.</summary>
|
||||
public const byte WarmActiveCritical = 100;
|
||||
|
||||
/// <summary>Warm Active, stale config with the DB reachable (spec §4.3).</summary>
|
||||
public const byte WarmActiveStale = 200;
|
||||
|
||||
/// <summary>Warm Active, healthy (spec §4.3): 240 + the 10-point Primary bonus.</summary>
|
||||
public const byte WarmActiveHealthy = 250;
|
||||
|
||||
/// <summary>
|
||||
/// The Warm ServiceLevel bands (Gitea #717 spec §4.3). <see cref="Compute"/> is unchanged and still serves
|
||||
/// <c>Unset</c> and <c>None</c>; this is used only under <c>OpcUa:Redundancy:Mode=Warm</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Invariant: every Standby level (≤ <see cref="WarmStandbyHealthy"/>) is below every level a data-serving
|
||||
/// Active node publishes (≥ <see cref="WarmActiveCritical"/>), so a client that picks the highest
|
||||
/// ServiceLevel never prefers a Standby. A member that is not Up/Joining, or whose own endpoint failed, is 0
|
||||
/// whatever its role (unchanged from <see cref="Compute"/>, lmxopcua#762).
|
||||
/// </remarks>
|
||||
/// <param name="h">The node's health facts; <see cref="NodeHealthInputs.IsDriverPrimary"/> is ignored — the
|
||||
/// Warm verdict <paramref name="isActive"/> replaces it.</param>
|
||||
/// <param name="isActive">The Warm role verdict (spec §4.1).</param>
|
||||
/// <returns>The ServiceLevel byte.</returns>
|
||||
public static byte ComputeWarm(NodeHealthInputs h, bool isActive)
|
||||
{
|
||||
if (h.MemberState is not (MemberStatus.Up or MemberStatus.Joining)) return 0;
|
||||
if (!h.EndpointListening) return 0;
|
||||
|
||||
var healthy = h.DbReachable && h.OpcUaProbeOk && !h.Stale;
|
||||
if (!isActive) return healthy ? WarmStandbyHealthy : WarmStandbyDegraded;
|
||||
if (healthy) return WarmActiveHealthy;
|
||||
return h.DbReachable && h.Stale ? WarmActiveStale : WarmActiveCritical;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,74 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
/// Broadcast whenever the cluster's redundancy topology changes (node up/down, role-leader
|
||||
/// change, partition heal). Subscribers compute their local OPC UA ServiceLevel from this.
|
||||
/// </summary>
|
||||
/// <param name="Nodes">One entry per cluster member.</param>
|
||||
/// <param name="CorrelationId">A random id per publish (diagnostics only; it carries no order).</param>
|
||||
/// <param name="Revision">The publisher's ordered revision (Gitea #717, spec §4.2). <c>null</c> from a pre-#717
|
||||
/// publisher during a rolling upgrade: such a snapshot still feeds the existing gates, but can never make a Warm
|
||||
/// node Active.</param>
|
||||
public sealed record RedundancyStateChanged(
|
||||
IReadOnlyList<NodeRedundancyState> Nodes,
|
||||
CorrelationId CorrelationId);
|
||||
CorrelationId CorrelationId,
|
||||
SnapshotRevision? Revision = null);
|
||||
|
||||
/// <summary>
|
||||
/// The order of <see cref="RedundancyStateChanged"/> snapshots (Gitea #717, spec §4.2), compared
|
||||
/// lexicographically. Within one cluster incarnation the singleton always lives on the oldest member, and every
|
||||
/// surviving member has a larger up-number than a departed oldest one, so <see cref="PublisherUpNumber"/> is
|
||||
/// monotonic across handovers; <see cref="IncarnationTick"/> orders a singleton restarted in the same process;
|
||||
/// <see cref="Sequence"/> orders the publishes of one incarnation.
|
||||
/// </summary>
|
||||
/// <param name="PublisherUpNumber">The <c>Member.UpNumber</c> of the node hosting the singleton.</param>
|
||||
/// <param name="IncarnationTick"><c>Environment.TickCount64</c> at the singleton's <c>PreStart</c>.</param>
|
||||
/// <param name="Sequence">Incremented on every publish, heartbeats included.</param>
|
||||
public sealed record SnapshotRevision(long PublisherUpNumber, long IncarnationTick, long Sequence)
|
||||
: IComparable<SnapshotRevision>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public int CompareTo(SnapshotRevision? other)
|
||||
{
|
||||
if (other is null) return 1;
|
||||
var c = PublisherUpNumber.CompareTo(other.PublisherUpNumber);
|
||||
if (c != 0) return c;
|
||||
c = IncarnationTick.CompareTo(other.IncarnationTick);
|
||||
return c != 0 ? c : Sequence.CompareTo(other.Sequence);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString() => $"{PublisherUpNumber}.{IncarnationTick}.{Sequence}";
|
||||
}
|
||||
|
||||
/// <summary>Why a snapshot was not accepted by a <see cref="SnapshotRevisionGate"/>.</summary>
|
||||
public enum SnapshotRejection
|
||||
{
|
||||
/// <summary>Accepted.</summary>
|
||||
None,
|
||||
|
||||
/// <summary>Its revision is not greater than the last accepted one (a late publish of an older incarnation).</summary>
|
||||
OlderRevision,
|
||||
|
||||
/// <summary>It carries no revision, while revisioned snapshots are being ordered.</summary>
|
||||
Unrevisioned,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The subscriber-side order rule of spec §4.2: accept a snapshot only if its revision is greater than the last
|
||||
/// accepted one. Not thread-safe; owned by one actor.
|
||||
/// </summary>
|
||||
public sealed class SnapshotRevisionGate
|
||||
{
|
||||
/// <summary>The last accepted revision; <c>null</c> until a revisioned snapshot was accepted.</summary>
|
||||
public SnapshotRevision? LastAccepted { get; private set; }
|
||||
|
||||
/// <summary>Decide one snapshot. An unrevisioned snapshot is reported <see cref="SnapshotRejection.Unrevisioned"/>
|
||||
/// and never advances the order (the caller decides whether the existing gates still use it).</summary>
|
||||
/// <param name="msg">The snapshot.</param>
|
||||
/// <returns><see cref="SnapshotRejection.None"/> when accepted.</returns>
|
||||
public SnapshotRejection Offer(RedundancyStateChanged msg)
|
||||
{
|
||||
if (msg.Revision is not { } revision) return SnapshotRejection.Unrevisioned;
|
||||
if (LastAccepted is { } last && revision.CompareTo(last) <= 0) return SnapshotRejection.OlderRevision;
|
||||
LastAccepted = revision;
|
||||
return SnapshotRejection.None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,6 +190,44 @@ public static class OtOpcUaTelemetry
|
||||
Meter.CreateCounter<long>("otopcua.opcua.condition_events.superseded", unit: "{delivery}",
|
||||
description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state.");
|
||||
|
||||
// ---------------- Warm redundancy (Gitea #717, spec §7) ----------------
|
||||
|
||||
private static int _warmRole = -1;
|
||||
|
||||
/// <summary>Records this process's Warm role for <see cref="WarmRole"/>: 1 Active, 0 Standby, −1 not Warm.</summary>
|
||||
/// <param name="value">The role value.</param>
|
||||
public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value);
|
||||
|
||||
/// <summary>The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm.</summary>
|
||||
public static readonly ObservableGauge<int> WarmRole =
|
||||
Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}",
|
||||
description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm.");
|
||||
|
||||
/// <summary>Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed).</summary>
|
||||
public static readonly Counter<long> WarmTransition =
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.warm_transition", unit: "{transition}",
|
||||
description: "Warm role transitions (direction=promote|demote, reason).");
|
||||
|
||||
/// <summary>Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned).</summary>
|
||||
public static readonly Counter<long> SnapshotRejected =
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}",
|
||||
description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned).");
|
||||
|
||||
/// <summary>The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.</summary>
|
||||
public static readonly Counter<long> RedundancyModeMismatch =
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.mode_mismatch", unit: "{apply}",
|
||||
description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.");
|
||||
|
||||
/// <summary>Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4).</summary>
|
||||
public static readonly Counter<long> StandbyProjectedNotifications =
|
||||
Meter.CreateCounter<long>("otopcua.opcua.standby.projected_notifications", unit: "{notification}",
|
||||
description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery.");
|
||||
|
||||
/// <summary>New alarm condition events not delivered on a Warm Standby (still stamped).</summary>
|
||||
public static readonly Counter<long> StandbySuppressedAlarmEvents =
|
||||
Meter.CreateCounter<long>("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}",
|
||||
description: "New alarm condition events suppressed on a Warm standby.");
|
||||
|
||||
// ---------------- Convenience helpers ----------------
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// The data-plane side of a Warm role transition (Gitea #717, spec §4.7). <c>OpcUaPublishActor</c> owns the
|
||||
/// verdict and drives these steps in a fixed order; the node manager implements them. Every member is
|
||||
/// non-blocking and idempotent. Under <see cref="OpcUaRedundancyMode.Unset"/> and
|
||||
/// <see cref="OpcUaRedundancyMode.None"/> nothing calls it.
|
||||
/// </summary>
|
||||
public interface IStandbyProjection
|
||||
{
|
||||
/// <summary>Opens (<c>false</c>) or closes (<c>true</c>) the per-tag read projection: while projected, every
|
||||
/// ordinary-tag <c>Value</c> read and sample answers <c>BadOutOfService</c>/null, and writes and Part 9 condition
|
||||
/// methods are refused <c>BadInvalidState</c>. The stored value keeps updating from acquisition.</summary>
|
||||
/// <param name="projected">Whether ordinary-tag values are projected.</param>
|
||||
void SetDataProjection(bool projected);
|
||||
|
||||
/// <summary>Switches the first-delivery decision (spec §4.6, D4): a Publish response delivered for the first time
|
||||
/// while <paramref name="standby"/> is recorded as Standby and its ordinary-tag notifications are rewritten.</summary>
|
||||
/// <param name="standby">Whether first deliveries are decided Standby.</param>
|
||||
void SetStandbyDelivery(bool standby);
|
||||
|
||||
/// <summary>Suppresses (<c>true</c>) or restores delivery of NEW alarm condition events (spec §4.5). Condition
|
||||
/// state keeps being stamped, so a later refresh carries current, ackable state.</summary>
|
||||
/// <param name="suppressed">Whether new alarm events are suppressed.</param>
|
||||
void SetAlarmSuppression(bool suppressed);
|
||||
|
||||
/// <summary>Queues each live, non-stood-down ordinary-tag data item one value, ignoring its filters: the current
|
||||
/// stored value (promotion) or <c>BadOutOfService</c>/null (demotion).</summary>
|
||||
/// <param name="projected">Whether to queue the projected value instead of the stored one.</param>
|
||||
/// <returns>The number of items queued to.</returns>
|
||||
int QueueCurrentToLiveItems(bool projected);
|
||||
|
||||
/// <summary>Runs a ConditionRefresh on every live event item (Server-object, folder and direct), through the
|
||||
/// existing refresh paths and their ACL decisions (spec §4.5 promotion refresh).</summary>
|
||||
/// <returns>The number of event items refreshed.</returns>
|
||||
int RefreshAllEventItems();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Late-binding <see cref="IStandbyProjection"/> (the <see cref="DeferredServiceLevelPublisher"/> pattern): the
|
||||
/// actor gets it at construction, the node manager only exists after SDK start. The latest requested
|
||||
/// projection / delivery / suppression state is retained and replayed on bind (the #719 lesson); the two
|
||||
/// queue/refresh actions are not replayed — before bind there are no items to act on.
|
||||
/// </summary>
|
||||
public sealed class DeferredStandbyProjection : IStandbyProjection
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private IStandbyProjection? _inner;
|
||||
private bool? _projected;
|
||||
private bool? _standby;
|
||||
private bool? _suppressed;
|
||||
|
||||
/// <summary>Bind (and replay the retained state), or unbind with <c>null</c>.</summary>
|
||||
/// <param name="inner">The live implementation, or null to unbind.</param>
|
||||
public void SetInner(IStandbyProjection? inner)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_inner = inner;
|
||||
if (inner is null) return;
|
||||
if (_projected is { } p) inner.SetDataProjection(p);
|
||||
if (_standby is { } s) inner.SetStandbyDelivery(s);
|
||||
if (_suppressed is { } a) inner.SetAlarmSuppression(a);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetDataProjection(bool projected)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_projected = projected;
|
||||
_inner?.SetDataProjection(projected);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetStandbyDelivery(bool standby)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_standby = standby;
|
||||
_inner?.SetStandbyDelivery(standby);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetAlarmSuppression(bool suppressed)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_suppressed = suppressed;
|
||||
_inner?.SetAlarmSuppression(suppressed);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public int QueueCurrentToLiveItems(bool projected)
|
||||
{
|
||||
IStandbyProjection? inner;
|
||||
lock (_gate) inner = _inner;
|
||||
return inner?.QueueCurrentToLiveItems(projected) ?? 0;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public int RefreshAllEventItems()
|
||||
{
|
||||
IStandbyProjection? inner;
|
||||
lock (_gate) inner = _inner;
|
||||
return inner?.RefreshAllEventItems() ?? 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// The parsed <c>OpcUa:Redundancy:Mode</c> (Gitea #717). Startup-static: changing it requires a restart.
|
||||
/// </summary>
|
||||
public enum OpcUaRedundancyMode
|
||||
{
|
||||
/// <summary>Not opted in (the default): byte-for-byte the pre-#717 server. Nothing in the redundancy metadata,
|
||||
/// ServiceLevel bands, projection or alarm delivery changes.</summary>
|
||||
Unset = 0,
|
||||
|
||||
/// <summary>Standalone: advertises <c>RedundancySupport=None</c> with no <c>ServerUriArray</c>; never projects.</summary>
|
||||
None = 1,
|
||||
|
||||
/// <summary>A 2-node pair with a degraded standby (spec §4): non-transparent Warm metadata, Standby bands below
|
||||
/// every Active level, ordinary-tag values projected <c>BadOutOfService</c> on the Standby, new alarm events
|
||||
/// suppressed there, and writes / Part 9 methods refused <c>BadInvalidState</c> there.</summary>
|
||||
Warm = 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The runtime half of <c>OpcUa:Redundancy</c> (Gitea #717): what the redundancy actors need. The Host builds it
|
||||
/// from the validated <c>OpcUaApplicationHostOptions</c> and registers it as a singleton; a node that registers
|
||||
/// none (a harness, an admin-only node) is <see cref="Unset"/>.
|
||||
/// </summary>
|
||||
/// <param name="Mode">The redundancy mode.</param>
|
||||
/// <param name="PromoteWhilePrimaryUnreachable">Warm only: let the survivor go Active while the Primary is
|
||||
/// unreachable but not yet downed (spec §4.1, owner answer Q2). Default on.</param>
|
||||
public sealed record WarmRedundancySettings(OpcUaRedundancyMode Mode, bool PromoteWhilePrimaryUnreachable = true)
|
||||
{
|
||||
/// <summary>The not-opted-in default.</summary>
|
||||
public static WarmRedundancySettings Unset { get; } = new(OpcUaRedundancyMode.Unset);
|
||||
|
||||
/// <summary>True when this node runs the Warm role contract.</summary>
|
||||
public bool IsWarm => Mode == OpcUaRedundancyMode.Warm;
|
||||
}
|
||||
Reference in New Issue
Block a user