diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs index 2be0b2612..fe9c5b4dc 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs @@ -62,4 +62,44 @@ public static class ServiceLevelCalculator return (byte)Math.Clamp(basis + (h.IsDriverPrimary ? 10 : 0), 0, 255); } + + /// Warm Standby, healthy (Gitea #717 spec §4.3): Degraded band. + public const byte WarmStandbyHealthy = 90; + + /// Warm Standby, stale or DB unreachable (spec §4.3): Degraded band. + public const byte WarmStandbyDegraded = 50; + + /// Warm Active, critically degraded (spec §4.3): the lowest level a data-serving Active node publishes. + public const byte WarmActiveCritical = 100; + + /// Warm Active, stale config with the DB reachable (spec §4.3). + public const byte WarmActiveStale = 200; + + /// Warm Active, healthy (spec §4.3): 240 + the 10-point Primary bonus. + public const byte WarmActiveHealthy = 250; + + /// + /// The Warm ServiceLevel bands (Gitea #717 spec §4.3). is unchanged and still serves + /// Unset and None; this is used only under OpcUa:Redundancy:Mode=Warm. + /// + /// + /// Invariant: every Standby level (≤ ) is below every level a data-serving + /// Active node publishes (≥ ), so a client that picks the highest + /// ServiceLevel never prefers a Standby. A member that is not Up/Joining, or whose own endpoint failed, is 0 + /// whatever its role (unchanged from , lmxopcua#762). + /// + /// The node's health facts; is ignored — the + /// Warm verdict replaces it. + /// The Warm role verdict (spec §4.1). + /// The ServiceLevel byte. + public static byte ComputeWarm(NodeHealthInputs h, bool isActive) + { + if (h.MemberState is not (MemberStatus.Up or MemberStatus.Joining)) return 0; + if (!h.EndpointListening) return 0; + + var healthy = h.DbReachable && h.OpcUaProbeOk && !h.Stale; + if (!isActive) return healthy ? WarmStandbyHealthy : WarmStandbyDegraded; + if (healthy) return WarmActiveHealthy; + return h.DbReachable && h.Stale ? WarmActiveStale : WarmActiveCritical; + } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Redundancy/RedundancyStateChanged.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Redundancy/RedundancyStateChanged.cs index 6b2163875..722733fb6 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Redundancy/RedundancyStateChanged.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Redundancy/RedundancyStateChanged.cs @@ -6,6 +6,74 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; /// Broadcast whenever the cluster's redundancy topology changes (node up/down, role-leader /// change, partition heal). Subscribers compute their local OPC UA ServiceLevel from this. /// +/// One entry per cluster member. +/// A random id per publish (diagnostics only; it carries no order). +/// The publisher's ordered revision (Gitea #717, spec §4.2). null from a pre-#717 +/// publisher during a rolling upgrade: such a snapshot still feeds the existing gates, but can never make a Warm +/// node Active. public sealed record RedundancyStateChanged( IReadOnlyList Nodes, - CorrelationId CorrelationId); + CorrelationId CorrelationId, + SnapshotRevision? Revision = null); + +/// +/// The order of snapshots (Gitea #717, spec §4.2), compared +/// lexicographically. Within one cluster incarnation the singleton always lives on the oldest member, and every +/// surviving member has a larger up-number than a departed oldest one, so is +/// monotonic across handovers; orders a singleton restarted in the same process; +/// orders the publishes of one incarnation. +/// +/// The Member.UpNumber of the node hosting the singleton. +/// Environment.TickCount64 at the singleton's PreStart. +/// Incremented on every publish, heartbeats included. +public sealed record SnapshotRevision(long PublisherUpNumber, long IncarnationTick, long Sequence) + : IComparable +{ + /// + public int CompareTo(SnapshotRevision? other) + { + if (other is null) return 1; + var c = PublisherUpNumber.CompareTo(other.PublisherUpNumber); + if (c != 0) return c; + c = IncarnationTick.CompareTo(other.IncarnationTick); + return c != 0 ? c : Sequence.CompareTo(other.Sequence); + } + + /// + public override string ToString() => $"{PublisherUpNumber}.{IncarnationTick}.{Sequence}"; +} + +/// Why a snapshot was not accepted by a . +public enum SnapshotRejection +{ + /// Accepted. + None, + + /// Its revision is not greater than the last accepted one (a late publish of an older incarnation). + OlderRevision, + + /// It carries no revision, while revisioned snapshots are being ordered. + Unrevisioned, +} + +/// +/// The subscriber-side order rule of spec §4.2: accept a snapshot only if its revision is greater than the last +/// accepted one. Not thread-safe; owned by one actor. +/// +public sealed class SnapshotRevisionGate +{ + /// The last accepted revision; null until a revisioned snapshot was accepted. + public SnapshotRevision? LastAccepted { get; private set; } + + /// Decide one snapshot. An unrevisioned snapshot is reported + /// and never advances the order (the caller decides whether the existing gates still use it). + /// The snapshot. + /// when accepted. + public SnapshotRejection Offer(RedundancyStateChanged msg) + { + if (msg.Revision is not { } revision) return SnapshotRejection.Unrevisioned; + if (LastAccepted is { } last && revision.CompareTo(last) <= 0) return SnapshotRejection.OlderRevision; + LastAccepted = revision; + return SnapshotRejection.None; + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs index 1bce3a838..54f5d5316 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs @@ -190,6 +190,44 @@ public static class OtOpcUaTelemetry Meter.CreateCounter("otopcua.opcua.condition_events.superseded", unit: "{delivery}", description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state."); + // ---------------- Warm redundancy (Gitea #717, spec §7) ---------------- + + private static int _warmRole = -1; + + /// Records this process's Warm role for : 1 Active, 0 Standby, −1 not Warm. + /// The role value. + public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value); + + /// The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm. + public static readonly ObservableGauge WarmRole = + Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}", + description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm."); + + /// Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed). + public static readonly Counter WarmTransition = + Meter.CreateCounter("otopcua.redundancy.warm_transition", unit: "{transition}", + description: "Warm role transitions (direction=promote|demote, reason)."); + + /// Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned). + public static readonly Counter SnapshotRejected = + Meter.CreateCounter("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}", + description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned)."); + + /// The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode. + public static readonly Counter RedundancyModeMismatch = + Meter.CreateCounter("otopcua.redundancy.mode_mismatch", unit: "{apply}", + description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode."); + + /// Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4). + public static readonly Counter StandbyProjectedNotifications = + Meter.CreateCounter("otopcua.opcua.standby.projected_notifications", unit: "{notification}", + description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery."); + + /// New alarm condition events not delivered on a Warm Standby (still stamped). + public static readonly Counter StandbySuppressedAlarmEvents = + Meter.CreateCounter("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}", + description: "New alarm condition events suppressed on a Warm standby."); + // ---------------- Convenience helpers ---------------- /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/OpcUa/IStandbyProjection.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/OpcUa/IStandbyProjection.cs new file mode 100644 index 000000000..26f5a3462 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/OpcUa/IStandbyProjection.cs @@ -0,0 +1,112 @@ +namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa; + +/// +/// The data-plane side of a Warm role transition (Gitea #717, spec §4.7). OpcUaPublishActor owns the +/// verdict and drives these steps in a fixed order; the node manager implements them. Every member is +/// non-blocking and idempotent. Under and +/// nothing calls it. +/// +public interface IStandbyProjection +{ + /// Opens (false) or closes (true) the per-tag read projection: while projected, every + /// ordinary-tag Value read and sample answers BadOutOfService/null, and writes and Part 9 condition + /// methods are refused BadInvalidState. The stored value keeps updating from acquisition. + /// Whether ordinary-tag values are projected. + void SetDataProjection(bool projected); + + /// Switches the first-delivery decision (spec §4.6, D4): a Publish response delivered for the first time + /// while is recorded as Standby and its ordinary-tag notifications are rewritten. + /// Whether first deliveries are decided Standby. + void SetStandbyDelivery(bool standby); + + /// Suppresses (true) or restores delivery of NEW alarm condition events (spec §4.5). Condition + /// state keeps being stamped, so a later refresh carries current, ackable state. + /// Whether new alarm events are suppressed. + void SetAlarmSuppression(bool suppressed); + + /// Queues each live, non-stood-down ordinary-tag data item one value, ignoring its filters: the current + /// stored value (promotion) or BadOutOfService/null (demotion). + /// Whether to queue the projected value instead of the stored one. + /// The number of items queued to. + int QueueCurrentToLiveItems(bool projected); + + /// Runs a ConditionRefresh on every live event item (Server-object, folder and direct), through the + /// existing refresh paths and their ACL decisions (spec §4.5 promotion refresh). + /// The number of event items refreshed. + int RefreshAllEventItems(); +} + +/// +/// Late-binding (the pattern): the +/// actor gets it at construction, the node manager only exists after SDK start. The latest requested +/// projection / delivery / suppression state is retained and replayed on bind (the #719 lesson); the two +/// queue/refresh actions are not replayed — before bind there are no items to act on. +/// +public sealed class DeferredStandbyProjection : IStandbyProjection +{ + private readonly object _gate = new(); + private IStandbyProjection? _inner; + private bool? _projected; + private bool? _standby; + private bool? _suppressed; + + /// Bind (and replay the retained state), or unbind with null. + /// The live implementation, or null to unbind. + public void SetInner(IStandbyProjection? inner) + { + lock (_gate) + { + _inner = inner; + if (inner is null) return; + if (_projected is { } p) inner.SetDataProjection(p); + if (_standby is { } s) inner.SetStandbyDelivery(s); + if (_suppressed is { } a) inner.SetAlarmSuppression(a); + } + } + + /// + public void SetDataProjection(bool projected) + { + lock (_gate) + { + _projected = projected; + _inner?.SetDataProjection(projected); + } + } + + /// + public void SetStandbyDelivery(bool standby) + { + lock (_gate) + { + _standby = standby; + _inner?.SetStandbyDelivery(standby); + } + } + + /// + public void SetAlarmSuppression(bool suppressed) + { + lock (_gate) + { + _suppressed = suppressed; + _inner?.SetAlarmSuppression(suppressed); + } + } + + /// + public int QueueCurrentToLiveItems(bool projected) + { + IStandbyProjection? inner; + lock (_gate) inner = _inner; + return inner?.QueueCurrentToLiveItems(projected) ?? 0; + } + + /// + public int RefreshAllEventItems() + { + IStandbyProjection? inner; + lock (_gate) inner = _inner; + return inner?.RefreshAllEventItems() ?? 0; + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/OpcUa/WarmRedundancySettings.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/OpcUa/WarmRedundancySettings.cs new file mode 100644 index 000000000..327722f05 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/OpcUa/WarmRedundancySettings.cs @@ -0,0 +1,36 @@ +namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa; + +/// +/// The parsed OpcUa:Redundancy:Mode (Gitea #717). Startup-static: changing it requires a restart. +/// +public enum OpcUaRedundancyMode +{ + /// Not opted in (the default): byte-for-byte the pre-#717 server. Nothing in the redundancy metadata, + /// ServiceLevel bands, projection or alarm delivery changes. + Unset = 0, + + /// Standalone: advertises RedundancySupport=None with no ServerUriArray; never projects. + None = 1, + + /// A 2-node pair with a degraded standby (spec §4): non-transparent Warm metadata, Standby bands below + /// every Active level, ordinary-tag values projected BadOutOfService on the Standby, new alarm events + /// suppressed there, and writes / Part 9 methods refused BadInvalidState there. + Warm = 2, +} + +/// +/// The runtime half of OpcUa:Redundancy (Gitea #717): what the redundancy actors need. The Host builds it +/// from the validated OpcUaApplicationHostOptions and registers it as a singleton; a node that registers +/// none (a harness, an admin-only node) is . +/// +/// The redundancy mode. +/// Warm only: let the survivor go Active while the Primary is +/// unreachable but not yet downed (spec §4.1, owner answer Q2). Default on. +public sealed record WarmRedundancySettings(OpcUaRedundancyMode Mode, bool PromoteWhilePrimaryUnreachable = true) +{ + /// The not-opted-in default. + public static WarmRedundancySettings Unset { get; } = new(OpcUaRedundancyMode.Unset); + + /// True when this node runs the Warm role contract. + public bool IsWarm => Mode == OpcUaRedundancyMode.Warm; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs index dd0ff9cb6..e93ad966e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs @@ -37,6 +37,11 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers private readonly Action? _broadcastOverride; private readonly TimeSpan _heartbeatInterval; private bool _dirty; + /// Gitea #717 spec §4.2: Environment.TickCount64 at , ordering a singleton + /// restarted in the same process. + private long _incarnationTick; + /// Gitea #717 spec §4.2: incremented on every publish, heartbeats included. + private long _sequence; /// Gets the timer scheduler for this actor. public ITimerScheduler Timers { get; set; } = null!; @@ -78,6 +83,7 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers /// protected override void PreStart() { + _incarnationTick = NextIncarnationTick(); _cluster.Subscribe( Self, ClusterEvent.InitialStateAsEvents, @@ -107,12 +113,63 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers _dirty = false; var snapshot = BuildSnapshot(); - var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId()); + var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId(), NextRevision()); if (_broadcastOverride is not null) _broadcastOverride(msg); else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(Topic, msg)); _log.Debug("Published RedundancyStateChanged with {Count} nodes", snapshot.Count); } + /// + /// The ordered revision of the next publish (Gitea #717 spec §4.2): the up-number of the member hosting this + /// singleton, this incarnation's start tick, and a per-publish sequence. Subscribers drop anything not newer, + /// so a late heartbeat from an old incarnation can no longer overwrite the new incarnation's snapshot. + /// + private SnapshotRevision NextRevision() + { + long upNumber; + try + { + upNumber = ReadUpNumber(_cluster.SelfMember) + ?? throw new InvalidOperationException("Akka.Cluster Member.UpNumber is not readable on this Akka version"); + } + catch (Exception ex) + { + // Not expected on a running singleton; an unordered 0 can never outrank a real publisher. + _log.Debug(ex, "RedundancyState: SelfMember up-number unavailable; publishing revision up-number 0"); + upNumber = 0; + } + + return new SnapshotRevision(upNumber, _incarnationTick, ++_sequence); + } + + private static long _lastIncarnationTick; + + /// Environment.TickCount64, made strictly increasing within the process so a singleton restarted + /// inside one tick still outranks its predecessor (whose sequence may be ahead of the new one's). + /// The incarnation tick. + public static long NextIncarnationTick() + { + while (true) + { + var last = Interlocked.Read(ref _lastIncarnationTick); + var next = Math.Max(Environment.TickCount64, last + 1); + if (Interlocked.CompareExchange(ref _lastIncarnationTick, next, last) == last) return next; + } + } + + private static readonly System.Reflection.PropertyInfo? UpNumberProperty = typeof(Member).GetProperty( + "UpNumber", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic); + + /// + /// A member's up-number — the order in which members became Up, the basis of Member.IsOlderThan and of + /// singleton placement. Akka.NET 1.5 keeps the property internal, so it is read by reflection; the + /// SnapshotRevisionTests pin that it resolves, so an Akka upgrade that renames it fails loudly. + /// + /// The member. + /// The up-number, or null if the property is unavailable. + public static int? ReadUpNumber(Member member) => + UpNumberProperty?.GetValue(member) is int value ? value : null; + /// The cluster role that marks a node as carrying the driver runtime. public const string DriverRole = RoleParser.Driver; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaApplicationHostOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaApplicationHostOptionsValidator.cs index 1f77d6b78..9b9347eec 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaApplicationHostOptionsValidator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaApplicationHostOptionsValidator.cs @@ -60,5 +60,10 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase< || lockout.MaxFailedAttemptsPerClient >= lockout.MaxFailedAttempts, "OpcUa:CredentialLockout:MaxFailedAttemptsPerClient must be 0 (= 2 x MaxFailedAttempts) or at least " + "OpcUa:CredentialLockout:MaxFailedAttempts"); + + // Gitea #717 spec §3: Hot/unknown modes fail start; Warm needs distinct peers that do not include this server's + // own ApplicationUri; None needs no peers. The effective URI is re-checked after an ApplicationConfigPath load. + foreach (var failure in (o.Redundancy ?? new OpcUaRedundancyOptions()).Check(o.ApplicationUri, o.PeerApplicationUris)) + builder.RequireThat(false, failure); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaRedundancyStartupGuard.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaRedundancyStartupGuard.cs new file mode 100644 index 000000000..3bc03bbff --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaRedundancyStartupGuard.cs @@ -0,0 +1,27 @@ +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// The one Gitea #717 startup rule the options validator cannot see: OpcUa:Redundancy:Mode=Warm needs the +/// driver role (the validator is registered only on driver nodes, so on any other node it would never run). +/// +public static class OpcUaRedundancyStartupGuard +{ + /// Fails host start when a node without the driver role is configured Warm. + /// The raw OpcUa:Redundancy:Mode value. + /// Whether this node carries the driver role. + /// Warm without the driver role. + public static void RequireDriverRoleForWarm(string? configuredMode, bool hasDriver) + { + if (hasDriver) return; + var options = new OpcUaRedundancyOptions { Mode = configuredMode ?? string.Empty }; + if (options.TryParseMode(out var mode) && mode == OpcUaRedundancyMode.Warm) + { + throw new InvalidOperationException( + "OpcUa:Redundancy:Mode is Warm on a node without the driver role; Warm redundancy needs the driver role " + + "(the OPC UA endpoint is served by driver nodes only). Remove the setting or add the driver role."); + } + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 79590528b..1aacbbd3c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -36,6 +36,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly OpcUaApplicationHostOptions _options; private readonly DeferredAddressSpaceSink _deferredSink; private readonly DeferredServiceLevelPublisher _deferredServiceLevel; + private readonly DeferredStandbyProjection? _deferredStandby; private readonly IOpcUaUserAuthenticator _userAuthenticator; private readonly IHistorianDataSource _historianDataSource; private readonly IDataPlaneAuthoritySource _dataPlaneAuthority; @@ -127,6 +128,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// The node's configured write bounds (DriverWrites:Timeout); the inbound /// write gateway's Ask is derived from them (lmxopcua#777). Optional: null ⇒ /// . + /// The late-bound Warm standby projection (Gitea #717) OpcUaPublishActor drives; + /// bound here once the SDK has started. Optional: null (and a non-Warm server) binds nothing. public OtOpcUaServerHostedService( IOptions options, DeferredAddressSpaceSink deferredSink, @@ -147,8 +150,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl OpcUaEndpointState? endpointState = null, EffectiveRoles? roles = null, NativeAlarmAckPreflightView? ackPreflightView = null, - DriverWriteBounds? writeBounds = null) + DriverWriteBounds? writeBounds = null, + DeferredStandbyProjection? deferredStandby = null) { + _deferredStandby = deferredStandby; ArgumentNullException.ThrowIfNull(aclOptions); ArgumentNullException.ThrowIfNull(time); @@ -402,6 +407,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl serverInternal, _loggerFactory.CreateLogger())); } + + // Gitea #717: the Warm standby projection. Bound right after the ServiceLevel so a transition the actor + // already decided replays onto the node manager; null (every non-Warm server) leaves the adapter unbound. + if (_server.CreateStandbyProjection() is { } standbyProjection) _deferredStandby?.SetInner(standbyProjection); } catch (Exception ex) { @@ -568,6 +577,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // half-disposed NodeManager. _deferredSink.SetSink(null); _deferredServiceLevel.SetInner(null); + _deferredStandby?.SetInner(null); // Restore the Null write gateway so a late client write doesn't Ask a stopping DriverHostActor. _server?.SetNodeWriteGateway(null); // Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 1a488c873..f42d5920b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -206,6 +206,10 @@ ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.OpcUaUserCertificatePaths.Configure( // lmxopcua#765: refresh/staleness of the cached API-key → Galaxy ack identity set (driver nodes consume it). builder.Services.AddGalaxyAckIdentityCache(builder.Configuration); +// Gitea #717 spec §3: Warm redundancy is a property of the OPC UA endpoint, which only a driver node serves. A Warm +// mode on a node without the driver role is a configuration error, not something to ignore silently. +OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(builder.Configuration["OpcUa:Redundancy:Mode"], hasDriver); + if (hasDriver) { builder.Services.AddOtOpcUaRuntime(); @@ -501,6 +505,11 @@ if (hasDriver) builder.Services.AddValidatedOptions( builder.Configuration, "OpcUa"); + // Gitea #717: the runtime half of OpcUa:Redundancy for the redundancy actors, and the late-bound standby + // projection the OPC UA hosted service binds once the SDK has started (same pattern as the ServiceLevel publisher). + builder.Services.AddSingleton(sp => sp.GetRequiredService>().Value.Redundancy.ToSettings()); + builder.Services.AddSingleton(); + // Lazy ActorSystem accessor so OtOpcUaServerHostedService can resolve the DistributedPubSub // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/EquipmentVariableState.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/EquipmentVariableState.cs index 6678be063..7feea2b09 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/EquipmentVariableState.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/EquipmentVariableState.cs @@ -6,8 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// Preserves the application's established equipment-write conversions across SDK 2.0's /// new pre-handler type validation. The owning callback remains the authorization and shape gate. /// -internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock) : BaseDataVariableState(parent) +internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock, WarmStandbyState? standby = null) + : BaseDataVariableState(parent) { + private readonly WarmStandbyState _standby = standby ?? WarmStandbyState.Off; + // Every field below is accessed under ownerLock. Tokens identify client attempts; the mutation // revision detects a driver publication that re-enters the callback before its optimistic commit. private long _nextWriteToken; @@ -82,6 +85,16 @@ internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock QualifiedName dataEncoding, ref Variant value, ref DateTimeUtc sourceTimestamp) { // Subscription samplers read the node directly, bypassing the manager's service lock. - lock (ownerLock) return base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp); + ServiceResult result; + lock (ownerLock) result = base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp); + + // Gitea #717 spec §4.4 (T0b): on a Warm Standby every value path — Read (IndexRange and DataEncoding included), + // the initial sample, the change fan-out, ResendData, AggregateFilter input — answers BadOutOfService/null. The + // stored Value is untouched and keeps updating from acquisition. The base read runs first, so an IndexRange or + // encoding error is still reported as such. Off (Unset/None/Active): returned unchanged. + if (ServiceResult.IsBad(result) || !_standby.DataProjected) return result; + value = Variant.Null; + sourceTimestamp = DateTimeUtc.MinValue; + return StatusCodes.BadOutOfService; } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OpcUaApplicationHost.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OpcUaApplicationHost.cs index 893462bad..2f945ec62 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OpcUaApplicationHost.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OpcUaApplicationHost.cs @@ -4,6 +4,7 @@ using Opc.Ua; using Opc.Ua.Configuration; using Opc.Ua.Identity; using Opc.Ua.Server; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Authorization; using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; @@ -174,6 +175,83 @@ public sealed class OpcUaApplicationHostOptions /// Replaces the SDK's own lockout, which also refused Anonymous; see . /// public OpcUaCredentialLockoutOptions CredentialLockout { get; set; } = new(); + + /// + /// Warm non-transparent redundancy (OpcUa:Redundancy:*, Gitea #717). The default Mode=Unset is + /// byte-for-byte today's server. Startup-static. + /// + public OpcUaRedundancyOptions Redundancy { get; set; } = new(); +} + +/// The OpcUa:Redundancy section (Gitea #717, spec §6). +public sealed class OpcUaRedundancyOptions +{ + /// + /// Unset (default) / None / Warm, case-insensitive. Hot is not advertisable + /// (spec §3) and, like any other value, fails host start. Bound as a string so a bad value is a readable + /// validation failure rather than a binder exception. + /// + public string Mode { get; set; } = nameof(OpcUaRedundancyMode.Unset); + + /// + /// Warm only: let the survivor go Active while the Primary is unreachable but not yet downed (spec §4.1, owner + /// answer Q2). Default true. + /// + public bool PromoteWhilePrimaryUnreachable { get; set; } = true; + + /// Parses . + /// The parsed mode. + /// false for Hot, numerics or any unknown value. + public bool TryParseMode(out OpcUaRedundancyMode mode) + { + var text = (Mode ?? string.Empty).Trim(); + if (text.Length == 0) text = nameof(OpcUaRedundancyMode.Unset); + if (!text.All(char.IsLetter)) { mode = default; return false; } + return Enum.TryParse(text, ignoreCase: true, out mode) && Enum.IsDefined(mode); + } + + /// The parsed mode; throws on an invalid one (the validator rejects it first). + public OpcUaRedundancyMode ParsedMode => TryParseMode(out var m) + ? m + : throw new InvalidOperationException( + $"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable)."); + + /// The runtime settings the redundancy actors read. + /// The settings. + public WarmRedundancySettings ToSettings() => new(ParsedMode, PromoteWhilePrimaryUnreachable); + + /// + /// The rules of spec §3 that need only configuration: Warm needs at least one peer, all URIs distinct + /// and the self URI not listed as a peer; None needs no peers. Returns the failure messages (empty when + /// valid). The effective self URI is checked again after an ApplicationConfigPath load. + /// + /// The self URI to check against. + /// The configured peer URIs. + /// The failures. + public IReadOnlyList Check(string? selfUri, IEnumerable? peers) + { + var failures = new List(); + if (!TryParseMode(out var mode)) + { + failures.Add($"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable: continuous acquisition alone does not meet Part 4 Hot)"); + return failures; + } + + var list = (peers ?? Array.Empty()).Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToList(); + if (mode == OpcUaRedundancyMode.None && list.Count > 0) + failures.Add("OpcUa:PeerApplicationUris must be empty when OpcUa:Redundancy:Mode is None (a standalone server has no peers)"); + if (mode == OpcUaRedundancyMode.Warm) + { + if (list.Count == 0) + failures.Add("OpcUa:Redundancy:Mode Warm requires at least one OpcUa:PeerApplicationUris entry"); + if (list.Distinct(StringComparer.Ordinal).Count() != list.Count) + failures.Add("OpcUa:PeerApplicationUris must not contain duplicates under OpcUa:Redundancy:Mode Warm"); + if (!string.IsNullOrWhiteSpace(selfUri) && list.Contains(selfUri.Trim(), StringComparer.Ordinal)) + failures.Add($"OpcUa:PeerApplicationUris must not list this server's own ApplicationUri '{selfUri}' under OpcUa:Redundancy:Mode Warm (each pair node needs a unique ApplicationUri)"); + } + + return failures; + } } /// @@ -289,7 +367,8 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable try { - _ = await BuildConfigurationAsync(cancellationToken); + var configuration = await BuildConfigurationAsync(cancellationToken); + ConfigureRedundancy(sdkServer, configuration.ApplicationUri ?? _options.ApplicationUri); await EnsureApplicationCertificateAsync(cancellationToken).ConfigureAwait(false); AttachUserAuthenticator(sdkServer); await _application.StartAsync(server, cancellationToken).ConfigureAwait(false); @@ -305,6 +384,31 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable _options.PublicHostname, _options.OpcUaPort); } + /// + /// Gitea #717 (spec §3): validate the redundancy mode against the EFFECTIVE self URI — the loaded + /// ApplicationConfigPath XML wins over OpcUa:ApplicationUri (c7787) — log the boot banner and hand + /// the startup-static metadata to the SDK server, which installs it in OnNodeManagerStartedAsync. + /// Unset hands nothing over, so the SDK's graph is untouched. + /// + /// The SDK server about to start. + /// The effective ApplicationConfiguration.ApplicationUri. + /// The redundancy configuration is invalid for the effective URI. + private void ConfigureRedundancy(OtOpcUaSdkServer server, string effectiveSelfUri) + { + var redundancy = _options.Redundancy ?? new OpcUaRedundancyOptions(); + var failures = redundancy.Check(effectiveSelfUri, _options.PeerApplicationUris); + if (failures.Count > 0) + throw new InvalidOperationException("OPC UA redundancy configuration is invalid: " + string.Join("; ", failures)); + + var mode = redundancy.ParsedMode; + var peers = (_options.PeerApplicationUris ?? new List()) + .Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToArray(); + _logger.LogInformation( + "OPC UA redundancy: Mode={Mode}, effective ApplicationUri={SelfUri}, peers=[{Peers}], PromoteWhilePrimaryUnreachable={Promote}", + mode, effectiveSelfUri, string.Join(", ", peers), redundancy.PromoteWhilePrimaryUnreachable); + server.ConfigureRedundancy(mode, effectiveSelfUri, peers); + } + private void AttachUserAuthenticator(OtOpcUaSdkServer server) { foreach (var type in new[] { UserTokenType.Anonymous, UserTokenType.UserName }) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index 7978efbc0..9214e1e29 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -161,6 +161,34 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 private FolderState? _root; private readonly ILogger _logger; + /// Initializes a new instance of the class with the OPC UA server and configuration. + /// The OPC UA server instance. + /// The application configuration. + /// Logger for the manager's swallowed-fault and skipped-wiring diagnostics; + /// null (tests) falls back to . + /// The Warm standby flags (Gitea #717); null (every non-Warm server) is + /// , which leaves every data path exactly as it was. + internal OtOpcUaNodeManager( + IServerInternal server, ApplicationConfiguration configuration, ILogger? logger, + WarmStandbyState? standby) + : this(server, configuration, logger) + { + _standby = standby ?? WarmStandbyState.Off; + if (_standby.WarmMode) + { + // Spec §4.6 item 1: ordinary-tag Value items are built by OrdinaryTagMonitoredItem, which marks each + // notification. Warm only — Unset/None keep the SDK's own manager. + m_monitoredItemManager.Dispose(); + m_monitoredItemManager = new OrdinaryTagItemManager(this.ToAsyncNodeManager(), server); + } + } + + /// The Warm standby flags (Gitea #717). unless the server is Warm. + private readonly WarmStandbyState _standby = WarmStandbyState.Off; + + /// Test observation: the Warm standby flags. + internal WarmStandbyState Standby => _standby; + /// Initializes a new instance of the class with the OPC UA server and configuration. /// The OPC UA server instance. /// The application configuration. @@ -1434,6 +1462,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 // the InstanceStateSnapshot captures the just-projected values. alarm.ClearChangeMasks(SystemContext, includeChildren: true); + // Gitea #717 spec §4.5: on a Warm Standby only the STAMP half above runs — the condition node is current and a + // later refresh carries a valid, ackable EventId — and the DELIVER half below does not. Suppressing here, before + // the snapshot and every SDK field projection, covers the Server-object, folder and direct paths, ACL on or + // off, by construction; overflow, filtered retain and the #838 deferral queue never see the event. The + // decision is made at report time: an event reported while Active is delivered even if that ends after + // demotion. + if (_standby.AlarmsSuppressed) + { + ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbySuppressedAlarmEvents.Add(1); + return; + } + try { // InstanceStateSnapshot is the IFilterTarget — a frozen copy of the condition's fields at fire @@ -2991,6 +3031,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 var gate = EvaluateAlarmCommandGate(identity, condition, operation); if (gate is not null) return gate; + // Gitea #717 owner answer Q4: a Warm Standby refuses every Part 9 condition method before it changes any state + // (nothing routed, nothing applied, no event). Unset/None keep today's acceptance. + if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal($"call {operation}"); + var cmd = new AlarmCommand( AlarmId: condition.NodeId.IdentifierAsString, Operation: operation, @@ -3032,6 +3076,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 var gate = EvaluateAlarmCommandGate(identity, condition, "Acknowledge"); if (gate is not null) return gate; + // Gitea #717 owner answer Q4: refused on a Warm Standby, ahead of (and whatever) the Primary gate below says. + if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("acknowledge the alarm"); + // PRIMARY GATE, after authorization (a caller who may not ack learns nothing about the // node's role). Only the Primary routes a native ack upstream (DriverHostActor applies the same verdict // to the route), so on any other node the ack is refused outright rather than applied locally and then @@ -3215,6 +3262,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 ?? EvaluateEquipmentWriteGate(identity, gatewayWired); if (gate is not null) return gate; + // Gitea #717 owner answer Q1: a Warm Standby refuses the write synchronously, AFTER authorization (a caller who + // may not write learns nothing about the node's role) and before anything reaches the driver. Nothing is + // applied, so nothing is reverted. Unset/None never set the flag and keep the #5 optimistic path. + if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("write the value"); + // Item A (synchronous structural fail-fast): reject a structurally-invalid write INLINE — return Bad // synchronously so the SDK never applies it (no optimistic-Good-then-revert round-trip + no needless // device dispatch). Runs AFTER the authz gate (so we never leak structure detail to an unauthorised @@ -4120,7 +4172,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 // The SDK exposes the flags separately (no CurrentReadWrite composite): ReadWrite is // CurrentRead | CurrentWrite, with the HistoryRead bit OR-ed in when historized. var access = ComposeAccessLevel(writable, historized, HistoryWritable(_historianDataUpdater)); - var variable = new EquipmentVariableState(parent, Lock) + var variable = new EquipmentVariableState(parent, Lock, _standby) { NodeId = new NodeId(variableNodeId, ns), BrowseName = new QualifiedName(variableNodeId, ns), @@ -5725,6 +5777,88 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 lock (Lock) return MonitoredNodes.ContainsKey(nodeId); } + /// + /// Gitea #717 spec §4.7 promotion step 2 / demotion step 5: queues one value to every live, non-stood-down + /// ordinary-tag Value item, ignoring its filters (the queue pattern of + /// — never detach or Disabled, which drops the queue). Promotion queues the current stored value read + /// through the item's own IndexRange/DataEncoding; demotion queues BadOutOfService/null. + /// + /// Queue the projected value (demotion) instead of the stored one (promotion). + /// The number of items queued to. + internal int QueueCurrentToLiveItems(bool projected) + { + var queued = 0; + foreach (var (item, tracked) in _liveItems) + { + if (tracked.Suppressed || tracked.Handle.Node is not EquipmentVariableState variable) continue; + if (item is not IDataChangeMonitoredItem2 { AttributeId: Attributes.Value } data) continue; + var now = UtcNow; + DataValue value; + if (projected) + { + value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, now); + } + else + { + var stored = new DataValue(); + ServiceResult read; + lock (Lock) read = variable.ReadAttribute(SystemContext, Attributes.Value, data.IndexRange, data.DataEncoding, ref stored); + value = ServiceResult.IsBad(read) + ? new DataValue(Variant.Null, read.StatusCode, DateTimeUtc.MinValue, now) + : new DataValue(stored.WrappedValue, stored.StatusCode, stored.SourceTimestamp, now); + } + + try + { + item.QueueValue(value, StatusCode.IsBad(value.StatusCode) ? new ServiceResult(value.StatusCode) : null, ignoreFilters: true); + queued++; + } + catch (Exception ex) + { + // A disabled item refuses the value; it re-samples the (projected or not) node when re-enabled. + _logger.LogDebug(ex, "OtOpcUaNodeManager: could not queue a Warm transition value to a monitored item"); + } + } + + return queued; + } + + /// + /// Gitea #717 spec §4.5 promotion refresh: a ConditionRefresh2 on every live event item — Server-object items + /// and the direct/folder items of this manager's notifiers — through the SDK's own subscription path, so each + /// client sees RefreshStart, the retained conditions (through this manager's ACL decision and the #854/#855 + /// cut), then RefreshEnd. Must not be called under Lock. + /// + /// The number of event items refreshed. + internal int RefreshAllEventItems() + { + var items = new List(_allEventsItems.Keys); + foreach (var (item, tracked) in _eventItems) + if (!tracked.Suppressed) items.Add(item); + if (items.Count == 0) return 0; + + var subscriptions = new Dictionary(); + foreach (var subscription in Server.SubscriptionManager.GetSubscriptions()) subscriptions[subscription.Id] = subscription; + + var refreshed = 0; + foreach (var item in items) + { + if (!subscriptions.TryGetValue(item.SubscriptionId, out var subscription)) continue; + try + { + subscription.ConditionRefresh2Async(item.Id).AsTask().GetAwaiter().GetResult(); + refreshed++; + } + catch (Exception ex) + { + // A subscription deleted mid-walk, or an item whose mode refuses a refresh: that client re-refreshes itself. + _logger.LogDebug(ex, "OtOpcUaNodeManager: Warm promotion refresh of event item {ItemId} failed", item.Id); + } + } + + return refreshed; + } + /// Test observability: how many data-change items the revalidation map tracks. internal int LiveItemCount => _liveItems.Count; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs index ceef01ed2..86bd5837f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs @@ -315,6 +315,160 @@ public sealed class OtOpcUaSdkServer : StandardServer /// public const byte StartupServiceLevel = 0; + private OpcUaRedundancyMode _redundancyMode = OpcUaRedundancyMode.Unset; + private string _selfUri = string.Empty; + private string[] _peerUris = []; + private WarmStandbyState _standby = WarmStandbyState.Off; + + /// The redundancy mode configured before start (Gitea #717). + public OpcUaRedundancyMode RedundancyMode => _redundancyMode; + + /// The Warm standby flags shared with the node manager. + internal WarmStandbyState Standby => _standby; + + /// The initial Server.ServiceLevel under Warm (spec §4.3: the NoData band, replacing the + /// SDK's 255 and this server's #885 value 0 until the first computed level). + public const byte WarmBootServiceLevel = 1; + + /// + /// Sets the startup-static redundancy metadata (Gitea #717 spec §3) before StartAsync. + /// (the default when this is never called) leaves the SDK graph and + /// every data path exactly as they are today. + /// + /// The validated mode. + /// The effective ApplicationConfiguration.ApplicationUri. + /// The peer URIs, in configured order. + public void ConfigureRedundancy(OpcUaRedundancyMode mode, string effectiveSelfUri, IReadOnlyList peerUris) + { + if (_otOpcUaNodeManager is not null) + throw new InvalidOperationException("Redundancy must be configured before the server starts."); + _redundancyMode = mode; + _selfUri = effectiveSelfUri; + _peerUris = peerUris.ToArray(); + _standby = new WarmStandbyState(mode == OpcUaRedundancyMode.Warm); + } + + /// + /// The data-plane transition steps a Warm OpcUaPublishActor drives (spec §4.7), or null when the server + /// is not Warm or has not started. + /// + /// The projection, or null. + public IStandbyProjection? CreateStandbyProjection() => + _standby.WarmMode && _otOpcUaNodeManager is { } nm ? new SdkStandbyProjection(nm, _standby) : null; + + /// Test seam: ids of the live subscriptions, for the decision record's dead-subscription sweep. + private IReadOnlySet LiveSubscriptionIds() + { + var ids = new HashSet(); + try + { + if (CurrentInstance?.SubscriptionManager is { } manager) + foreach (var subscription in manager.GetSubscriptions()) ids.Add(subscription.Id); + } + catch (Exception) + { + // Shutdown race: an empty set only drops records, which re-decide (fail-safe) at their next delivery. + } + + return ids; + } + + private long _publishesSinceSweep; + + /// + public override async ValueTask PublishAsync(SecureChannelContext secureChannelContext, + RequestHeader? requestHeader, ArrayOf subscriptionAcknowledgements, + RequestLifetime requestLifetime) + { + var response = await base.PublishAsync(secureChannelContext, requestHeader, subscriptionAcknowledgements, requestLifetime) + .ConfigureAwait(false); + if (_standby.Delivery is not { } record || response?.NotificationMessage is not { } message) return response!; + + if (message.NotificationData.Count > 0) + { + response.NotificationMessage = Decide(record, response.SubscriptionId, message); + var available = new List(response.AvailableSequenceNumbers.Count); + for (var i = 0; i < response.AvailableSequenceNumbers.Count; i++) available.Add(response.AvailableSequenceNumbers[i]); + record.Prune(response.SubscriptionId, available, message.SequenceNumber); + } + + if (Interlocked.Increment(ref _publishesSinceSweep) % 1024 == 0) record.SweepDead(LiveSubscriptionIds()); + return response; + } + + /// + public override async ValueTask RepublishAsync(SecureChannelContext secureChannelContext, + RequestHeader? requestHeader, uint subscriptionId, uint retransmitSequenceNumber, RequestLifetime requestLifetime) + { + var response = await base.RepublishAsync(secureChannelContext, requestHeader, subscriptionId, retransmitSequenceNumber, + requestLifetime).ConfigureAwait(false); + if (_standby.Delivery is { } record && response?.NotificationMessage is { NotificationData.Count: > 0 } message) + response.NotificationMessage = Decide(record, subscriptionId, message); + return response!; + } + + /// The recorded (or, first time, current) decision for one delivery, applied to the SDK's fresh clone. + private NotificationMessage Decide(FirstDeliveryRecord record, uint subscriptionId, NotificationMessage message) + { + if (!record.Decide(subscriptionId, message.SequenceNumber)) return message; + var projected = StandbyNotificationRewriter.Project(message, out var rewritten); + if (rewritten > 0) + { + ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbyProjectedNotifications.Add(rewritten); + } + + return projected; + } + + /// + /// Installs the standard redundancy metadata (Gitea #717 spec §3, T0a): under Warm + /// Server.ServerRedundancy becomes a NonTransparentRedundancyType whose read-only + /// ServerUriArray (i=11314) is [self, peers…], RedundancySupport is Warm and + /// ServiceLevel boots at ; under None RedundancySupport is + /// None and the SDK's orphan i=11314 is deleted, so it is absent by NodeId as well as by browse. + /// Unset touches nothing. + /// + private async ValueTask InstallRedundancyMetadataAsync(IServerInternal server, CancellationToken cancellationToken) + { + if (_redundancyMode == OpcUaRedundancyMode.Unset) return; + var serverObject = server.ServerObject + ?? throw new InvalidOperationException("The Server object is unavailable during server startup."); + var context = server.DefaultSystemContext; + var redundancy = serverObject.ServerRedundancy + ?? throw new InvalidOperationException("Server.ServerRedundancy is unavailable during server startup."); + + if (_redundancyMode == OpcUaRedundancyMode.None) + { + redundancy.RedundancySupport!.Value = RedundancySupport.None; + redundancy.RedundancySupport.ClearChangeMasks(context, false); + // T0a finding: the SDK's predefined i=11314 survives as an orphan (unreferenced, but Good [] by NodeId). + if (server.DiagnosticsNodeManager is AsyncCustomNodeManager diagnostics) + await diagnostics.DeleteNodeAsync(context, VariableIds.Server_ServerRedundancy_ServerUriArray, cancellationToken) + .ConfigureAwait(false); + return; + } + + var replacer = server.DiagnosticsNodeManager as IPredefinedNodeSubtypeReplacer + ?? throw new InvalidOperationException("The diagnostics node manager cannot replace predefined node subtypes."); + var replaced = (NonTransparentRedundancyState)await replacer.ReplacePredefinedInstanceSubtypeAsync(context, redundancy, + context.CreateInstanceOfNonTransparentRedundancyType(), + new Dictionary + { + [new QualifiedName(BrowseNames.ServerUriArray, 0)] = VariableIds.Server_ServerRedundancy_ServerUriArray, + }, + node => serverObject.ServerRedundancy = (ServerRedundancyState)node, cancellationToken).ConfigureAwait(false); + replaced.RedundancySupport!.Value = RedundancySupport.Warm; + replaced.ServerUriArray!.Value = (ArrayOf)new[] { _selfUri }.Concat(_peerUris).ToArray(); + replaced.ServerUriArray.AccessLevel = replaced.ServerUriArray.UserAccessLevel = AccessLevels.CurrentRead; + replaced.ClearChangeMasks(context, true); + if (serverObject.ServiceLevel is { } serviceLevel) + { + serviceLevel.Value = WarmBootServiceLevel; + serviceLevel.Timestamp = DateTime.UtcNow; + serviceLevel.ClearChangeMasks(context, includeChildren: false); + } + } + /// protected override async ValueTask OnNodeManagerStartedAsync( IServerInternal server, CancellationToken cancellationToken = default) @@ -329,6 +483,7 @@ public sealed class OtOpcUaSdkServer : StandardServer serviceLevel.Timestamp = DateTime.UtcNow; serviceLevel.ClearChangeMasks(server.DefaultSystemContext, includeChildren: false); } + await InstallRedundancyMetadataAsync(server, cancellationToken).ConfigureAwait(false); var auditType = server.DiagnosticsNodeManager? .FindPredefinedNode(ObjectTypeIds.AuditWriteUpdateEventType) ?? throw new InvalidOperationException("AuditWriteUpdateEventType is unavailable during server startup."); @@ -360,7 +515,7 @@ public sealed class OtOpcUaSdkServer : StandardServer protected override MasterNodeManager CreateMasterNodeManager( IServerInternal server, ApplicationConfiguration configuration) { - _otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger); + _otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger, _standby); return new MasterNodeManager(server, configuration, dynamicNamespaceUri: null, _otOpcUaNodeManager); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/WarmStandby.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/WarmStandby.cs new file mode 100644 index 000000000..af53baaf6 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/WarmStandby.cs @@ -0,0 +1,388 @@ +using Microsoft.Extensions.Logging; +using Opc.Ua; +using Opc.Ua.Server; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; + +/// +/// The node-manager-wide Warm standby flags (Gitea #717, spec §4.4–§4.6). One instance per server. Under +/// Unset/None is false and nothing ever sets a flag, so every reader sees +/// today's behaviour. Under Warm the node starts Standby (all flags set) until the redundancy layer +/// promotes it. +/// +internal sealed class WarmStandbyState +{ + private volatile bool _dataProjected; + private volatile bool _alarmsSuppressed; + + /// Initializes the state. + /// Whether this server runs the Warm role contract. + public WarmStandbyState(bool warmMode) + { + WarmMode = warmMode; + _dataProjected = warmMode; + _alarmsSuppressed = warmMode; + Delivery = warmMode ? new FirstDeliveryRecord(standby: true) : null; + } + + /// The not-Warm state (no projection, no suppression, no decision record). + public static WarmStandbyState Off { get; } = new(false); + + /// Whether this server runs the Warm role contract. + public bool WarmMode { get; } + + /// Ordinary-tag reads/samples project BadOutOfService; writes and Part 9 methods are refused. + public bool DataProjected + { + get => _dataProjected; + set => _dataProjected = WarmMode && value; + } + + /// New alarm condition events are not delivered (still stamped). + public bool AlarmsSuppressed + { + get => _alarmsSuppressed; + set => _alarmsSuppressed = WarmMode && value; + } + + /// Whether a Warm Standby must refuse a write or a Part 9 method (owner answers Q1/Q4). + public bool RefusesMutations => WarmMode && _dataProjected; + + /// The first-delivery decision record; null when not Warm. + public FirstDeliveryRecord? Delivery { get; } + + /// The refusal returned to a write or a Part 9 method on a Warm Standby. + public static ServiceResult StandbyRefusal(string what) => new(StatusCodes.BadInvalidState, + new LocalizedText($"This server is the Warm redundancy standby; {what} on the active server (the one with the higher ServiceLevel).")); +} + +/// +/// The ordinary-tag marker (spec §4.6 item 1): set on , a helper +/// reference the binary encoder never writes and the SDK's per-delivery CoreUtils.Clone copies +/// (pinned by WarmStandbySdkFieldContractTests). Valid only for the notification whose +/// and it captured, so a stale marker left on a pooled payload +/// (2.0.1's Initialize() does not clear the helper) fails . +/// +/// Derives from only because that is the helper's declared type. +internal sealed class OrdinaryTagMarker : NotificationMessage +{ + /// Initializes the marker. + /// The notification's client handle when it was built. + /// The notification's final value when it was built. + public OrdinaryTagMarker(uint clientHandle, DataValue value) + { + ClientHandle = clientHandle; + Value = value; + } + + /// The client handle captured at build time. + public uint ClientHandle { get; } + + /// The value captured at build time. + public DataValue Value { get; } + + /// Whether is the notification this marker was made for. + /// A delivered notification. + /// True for a match on client handle and value. + public bool Matches(MonitoredItemNotification n) => n.ClientHandle == ClientHandle && Equals(n.Value, Value); + + /// Whether carries a valid ordinary-tag marker. + /// A delivered notification. + /// True when it is an ordinary-tag notification. + public static bool IsOrdinaryTag(MonitoredItemNotification n) => n.Message is OrdinaryTagMarker m && m.Matches(n); +} + +/// +/// The per-message first-delivery decision (spec §4.6 item 2, D4), keyed by +/// (SubscriptionId, SequenceNumber) — never by object: SDK 2.0.1 hands every Publish and Republish a fresh +/// clone of the cached message. The first delivery of a sequence number records Active or Standby; every later +/// delivery (Republish included) applies the recorded decision. One lock orders a decision against a role flip; +/// it is never held across an await or a projection. +/// +internal sealed class FirstDeliveryRecord +{ + /// Recorded messages kept per subscription beyond what AvailableSequenceNumbers prunes. + internal const int MaxPerSubscription = 1024; + + /// Subscriptions kept before the least-recently-used one is dropped (dead subscriptions). + internal const int MaxSubscriptions = 8192; + + private readonly object _gate = new(); + private readonly Dictionary _subscriptions = new(); + private bool _standby; + private long _touch; + + /// Initializes the record. + /// The initial delivery state (a Warm node boots Standby). + public FirstDeliveryRecord(bool standby) => _standby = standby; + + /// Test seam: runs inside the lock after the decision is read and before it is recorded. + internal Action? DecisionBarrier { get; set; } + + /// The current first-delivery state. + public bool Standby + { + get { lock (_gate) return _standby; } + } + + /// Flips the first-delivery state (the D4 boundary). Waits for a decision in progress. + /// Whether later first deliveries are Standby. + public void SetStandby(bool standby) + { + lock (_gate) _standby = standby; + } + + /// The decision for one delivery: the recorded one, or (first delivery) the current state, recorded. + /// The subscription. + /// The message's sequence number. + /// True when the delivery must be projected (Standby). + public bool Decide(uint subscriptionId, uint sequenceNumber) + { + lock (_gate) + { + if (!_subscriptions.TryGetValue(subscriptionId, out var sub)) + { + if (_subscriptions.Count >= MaxSubscriptions) EvictLeastRecentlyUsed(); + _subscriptions[subscriptionId] = sub = new SubscriptionRecord(); + } + + sub.Touched = ++_touch; + if (sub.Decisions.TryGetValue(sequenceNumber, out var recorded)) return recorded; + var standby = _standby; + DecisionBarrier?.Invoke(); + sub.Decisions[sequenceNumber] = standby; + sub.Order.Enqueue(sequenceNumber); + while (sub.Order.Count > MaxPerSubscription && sub.Order.TryDequeue(out var oldest)) sub.Decisions.Remove(oldest); + return standby; + } + } + + /// Drops the decisions of messages the subscription no longer retains (they can never be republished). + /// The subscription the Publish response belongs to. + /// The response's AvailableSequenceNumbers. + /// The sequence number just delivered (kept: it becomes available for Republish). + public void Prune(uint subscriptionId, IReadOnlyCollection available, uint justDelivered) + { + lock (_gate) + { + if (!_subscriptions.TryGetValue(subscriptionId, out var sub)) return; + var keep = new HashSet(available) { justDelivered }; + var kept = new Queue(); + while (sub.Order.TryDequeue(out var seq)) + { + if (keep.Contains(seq)) kept.Enqueue(seq); + else sub.Decisions.Remove(seq); + } + + sub.Order.Clear(); + foreach (var seq in kept) sub.Order.Enqueue(seq); + } + } + + /// Drops every record of subscriptions not in . + /// The ids of live subscriptions. + public void SweepDead(IReadOnlySet live) + { + lock (_gate) + { + foreach (var id in _subscriptions.Keys.Where(id => !live.Contains(id)).ToList()) _subscriptions.Remove(id); + } + } + + /// Recorded decisions (test observation). + internal int Count + { + get { lock (_gate) return _subscriptions.Values.Sum(s => s.Decisions.Count); } + } + + /// Recorded subscriptions (test observation). + internal int SubscriptionCount + { + get { lock (_gate) return _subscriptions.Count; } + } + + private void EvictLeastRecentlyUsed() + { + var victim = _subscriptions.MinBy(kv => kv.Value.Touched).Key; + _subscriptions.Remove(victim); + } + + private sealed class SubscriptionRecord + { + public readonly Dictionary Decisions = new(); + public readonly Queue Order = new(); + public long Touched; + } +} + +/// +/// Rewrites a Standby delivery (spec §4.6): every marked ordinary-tag notification whose status is not Bad becomes +/// BadOutOfService/null with its server timestamp kept. Builds new objects; the message handed in (the SDK's +/// fresh per-delivery clone) and, a fortiori, the SDK's cached copy are never mutated. +/// +internal static class StandbyNotificationRewriter +{ + /// Projects , or returns it unchanged when nothing needs rewriting. + /// A delivered message. + /// The number of notifications rewritten. + /// The message to deliver. + public static NotificationMessage Project(NotificationMessage message, out int rewritten) + { + rewritten = 0; + if (message is null || message.NotificationData.Count == 0) return message!; + + var data = new List(message.NotificationData.Count); + var changed = false; + for (var i = 0; i < message.NotificationData.Count; i++) + { + var eo = message.NotificationData[i]; + if (eo.TryGetValue(out var dcn) && dcn is not null) + { + List? items = null; + for (var j = 0; j < dcn.MonitoredItems.Count; j++) + { + var n = dcn.MonitoredItems[j]; + if (OrdinaryTagMarker.IsOrdinaryTag(n) && !StatusCode.IsBad(n.Value.StatusCode)) + { + items ??= CopyPrefix(dcn, j); + items.Add(new MonitoredItemNotification + { + ClientHandle = n.ClientHandle, + Value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, n.Value.ServerTimestamp), + }); + rewritten++; + } + else + { + items?.Add(n); + } + } + + if (items is not null) + { + changed = true; + data.Add(new ExtensionObject(new DataChangeNotification { MonitoredItems = items, DiagnosticInfos = dcn.DiagnosticInfos })); + continue; + } + } + + data.Add(eo); + } + + if (!changed) return message; + return new NotificationMessage { SequenceNumber = message.SequenceNumber, PublishTime = message.PublishTime, NotificationData = data }; + } + + private static List CopyPrefix(DataChangeNotification dcn, int count) + { + var list = new List(dcn.MonitoredItems.Count); + for (var k = 0; k < count; k++) list.Add(dcn.MonitoredItems[k]); + return list; + } +} + +/// +/// A sampled data-change item for an ordinary-tag Value (spec §4.6 item 1): after the SDK's own per-value +/// publish (so the timestamp filter has run) it marks each notification it produced with an +/// . Everything else is the SDK's . +/// +internal sealed class OrdinaryTagMonitoredItem : MonitoredItem +{ + /// Initializes the item exactly as does. + public OrdinaryTagMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager, NodeHandle handle, uint subscriptionId, + uint id, MonitoredItemCreateRequest request, DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn, + MonitoringFilter? filter, Opc.Ua.Range? range, double samplingInterval, uint queueSize, bool durable) + : base(server, nodeManager, handle, subscriptionId, id, request.ItemToMonitor, diagnosticsMasks, timestampsToReturn, + request.MonitoringMode, request.RequestedParameters.ClientHandle, filter, filter, range, samplingInterval, queueSize, + request.RequestedParameters.DiscardOldest, 0, durable) + { + } + + /// + public override bool Publish(OperationContext context, Queue notifications, + Queue diagnostics, uint maxNotificationsPerPublish, ILogger logger) + { + var local = new Queue(); + var localDiagnostics = new Queue(); + var more = base.Publish(context, local, localDiagnostics, maxNotificationsPerPublish, logger); + while (local.Count > 0) + { + var n = local.Dequeue(); + n.Message = new OrdinaryTagMarker(n.ClientHandle, n.Value); + notifications.Enqueue(n); + if (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue()); + } + + while (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue()); + return more; + } +} + +/// +/// Installs for ordinary-tag () +/// Value items only. Re-implements on the SDK's +/// public, non-sealed manager, so CustomNodeManager2's is MonitoredNodeMonitoredItemManager checks and +/// its internal lifecycle interface keep working (T0c; the 2.0.1 MonitoredItemFactory hook exists only on +/// AsyncCustomNodeManager). Installed only under Warm; Unset/None keep the SDK's manager. +/// +internal sealed class OrdinaryTagItemManager(IAsyncNodeManager nodeManager, IServerInternal server) + : MonitoredNodeMonitoredItemManager(nodeManager, server), IMonitoredItemManager +{ + /// + ISampledDataChangeMonitoredItem IMonitoredItemManager.CreateMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager, + ServerSystemContext context, NodeHandle handle, uint subscriptionId, double publishingInterval, + DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn, MonitoredItemCreateRequest itemToCreate, + Opc.Ua.Range euRange, MonitoringFilter filterToUse, double samplingInterval, uint revisedQueueSize, bool createDurable, + MonitoredItemIdFactory monitoredItemIdFactory, Func addNodeToComponentCache) + { + if (handle.Node is not EquipmentVariableState || itemToCreate.ItemToMonitor.AttributeId != Attributes.Value) + { + return CreateMonitoredItem(server, nodeManager, context, handle, subscriptionId, publishingInterval, diagnosticsMasks, + timestampsToReturn, itemToCreate, euRange, filterToUse, samplingInterval, revisedQueueSize, createDurable, + monitoredItemIdFactory, addNodeToComponentCache); + } + + // Mirrors MonitoredNodeMonitoredItemManager.CreateMonitoredItem in exact tagged 2.0.1, swapping only the item type. + var cachedNode = addNodeToComponentCache(context, handle, handle.Node); + if (!MonitoredNodes.TryGetValue(handle.Node.NodeId, out var monitoredNode)) + { + MonitoredNodes[handle.Node.NodeId] = monitoredNode = new MonitoredNode2(nodeManager, server, cachedNode, + nodeManager.IsMultipleEventConsumerNode(cachedNode.NodeId)); + } + + handle.Node = monitoredNode.Node; + handle.MonitoredNode = monitoredNode; + uint id; + do + { + id = monitoredItemIdFactory.GetNextId(); + } + while (!MonitoredItems.TryAdd(id, null!)); + + var item = new OrdinaryTagMonitoredItem(server, nodeManager, handle, subscriptionId, id, itemToCreate, diagnosticsMasks, + timestampsToReturn, filterToUse, euRange, samplingInterval, revisedQueueSize, createDurable); + monitoredNode.Add((IDataChangeMonitoredItem2)item); + MonitoredItems[id] = item; + return item; + } +} + +/// The a Warm server hands the redundancy layer. +internal sealed class SdkStandbyProjection(OtOpcUaNodeManager nodeManager, WarmStandbyState state) + : ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection +{ + /// + public void SetDataProjection(bool projected) => state.DataProjected = projected; + + /// + public void SetStandbyDelivery(bool standby) => state.Delivery?.SetStandby(standby); + + /// + public void SetAlarmSuppression(bool suppressed) => state.AlarmsSuppressed = suppressed; + + /// + public int QueueCurrentToLiveItems(bool projected) => nodeManager.QueueCurrentToLiveItems(projected); + + /// + public int RefreshAllEventItems() => nodeManager.RefreshAllEventItems(); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs index 882d3d408..40cb890c5 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs @@ -258,6 +258,52 @@ public static class DeploymentArtifact } } + /// + /// Gitea #717 spec §3: the ServerCluster.RedundancyMode of this node's cluster as the artifact carries it + /// (None/Warm/Hot), for the deploy-time cross-check against OpcUa:Redundancy:Mode. + /// A single-cluster artifact answers for its only cluster; a multi-cluster one for the cluster of this node's + /// ClusterNode row. Null when the artifact does not say (empty, malformed, no matching cluster). + /// + /// The deployment artifact. + /// This node's host:port id. + /// The mode name, or null. + public static string? ReadClusterRedundancyMode(ReadOnlySpan blob, string nodeId) + { + if (blob.IsEmpty) return null; + try + { + using var parsed = Open(blob.ToArray()); + var root = parsed.Root; + if (!root.TryGetProperty("Clusters", out var clusters) || clusters.ValueKind != JsonValueKind.Array) return null; + var scope = parsed.ResolveClusterScope(nodeId); + JsonElement? match = null; + foreach (var cluster in clusters.EnumerateArray()) + { + if (cluster.ValueKind != JsonValueKind.Object) continue; + if (scope.Mode == ClusterFilterMode.None) { match = cluster; break; } + if (scope.Mode == ClusterFilterMode.ScopeTo + && string.Equals(cluster.TryGetProperty("ClusterId", out var cid) && cid.ValueKind == JsonValueKind.String ? cid.GetString() : null, scope.ClusterId, StringComparison.OrdinalIgnoreCase)) + { + match = cluster; + break; + } + } + + if (match is not { } m || !m.TryGetProperty("RedundancyMode", out var mode)) return null; + return mode.ValueKind switch + { + JsonValueKind.String => mode.GetString(), + JsonValueKind.Number when mode.TryGetInt32(out var n) && Enum.IsDefined(typeof(Configuration.Enums.RedundancyMode), n) + => ((Configuration.Enums.RedundancyMode)n).ToString(), + _ => null, + }; + } + catch (JsonException) + { + return null; + } + } + /// /// Resolve how a node should scope a deployment artifact to its own ClusterId. Single-cluster /// (or legacy) artifacts resolve to so every existing diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 352ef3b7a..7affe2891 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -789,7 +789,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS DriverWriteBounds? writeBounds = null, TimeProvider? writeClock = null, IScriptRunner? scriptedAlarmRunner = null, - TimeSpan? childShutdownTimeout = null) => + TimeSpan? childShutdownTimeout = null, + bool enforceSnapshotOrdering = false) => // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an // expression tree. Six IActorRef? parameters and several interface-typed ones mean a // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds @@ -803,7 +804,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView, sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay, sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView, - scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout)); + scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout, + enforceSnapshotOrdering)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -938,8 +940,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS DriverWriteBounds? writeBounds = null, TimeProvider? writeClock = null, IScriptRunner? scriptedAlarmRunner = null, - TimeSpan? childShutdownTimeout = null) + TimeSpan? childShutdownTimeout = null, + bool enforceSnapshotOrdering = false) { + _snapshotOrdering = enforceSnapshotOrdering ? new SnapshotRevisionGate() : null; _childShutdownTimeout = childShutdownTimeout; _scriptedAlarmRunner = scriptedAlarmRunner; _writeBounds = writeBounds ?? DriverWriteBounds.Default; @@ -2552,8 +2556,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// the cached role unchanged; when this node has a real driver peer (count > 1) that omission is the /// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors /// ScriptedAlarmHostActor.OnRedundancyStateChanged / OpcUaPublishActor. + /// Gitea #717: the snapshot order gate under Warm; null (no ordering, today's behaviour) otherwise. + private readonly SnapshotRevisionGate? _snapshotOrdering; + private void OnRedundancyStateChanged(RedundancyStateChanged msg) { + // Gitea #717 spec §4.2, under OpcUa:Redundancy:Mode=Warm only (Unset/None keep today's unconditional replace): + // a snapshot older than the last accepted one — a late heartbeat of an old singleton incarnation — is dropped, + // so it cannot overwrite the new incarnation's roles. An unrevisioned snapshot (pre-#717 publisher) still feeds + // the gates, as before. + if (_snapshotOrdering?.Offer(msg) == SnapshotRejection.OlderRevision) + { + OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair("reason", "older_revision")); + return; + } + _redundancySnapshotReceived = true; var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode); if (local is not null) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs index f90d229c4..d87737dff 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs @@ -14,6 +14,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Authorization; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Health; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; @@ -326,7 +327,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers bool dbLess = false, IDataPlaneAuthoritySink? dataPlaneAuthority = null, Func? utcNow = null, - IOpcUaEndpointState? endpointState = null) => + IOpcUaEndpointState? endpointState = null, + WarmRedundancySettings? redundancy = null, + IStandbyProjection? standbyProjection = null) => Akka.Actor.Props.Create(() => new OpcUaPublishActor( sink ?? NullOpcUaAddressSpaceSink.Instance, serviceLevel ?? NullServiceLevelPublisher.Instance, @@ -341,7 +344,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers dbLess, dataPlaneAuthority, utcNow, - endpointState)).WithDispatcher(DispatcherId); + endpointState, + redundancy, + standbyProjection)).WithDispatcher(DispatcherId); /// Test-only Props that omits the pinned-dispatcher requirement and skips the /// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster. @@ -385,7 +390,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers bool dbLess = false, IDataPlaneAuthoritySink? dataPlaneAuthority = null, Func? utcNow = null, - IOpcUaEndpointState? endpointState = null) => + IOpcUaEndpointState? endpointState = null, + WarmRedundancySettings? redundancy = null, + IStandbyProjection? standbyProjection = null) => Akka.Actor.Props.Create(() => new OpcUaPublishActor( sink ?? NullOpcUaAddressSpaceSink.Instance, serviceLevel ?? NullServiceLevelPublisher.Instance, @@ -400,7 +407,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers dbLess, dataPlaneAuthority, utcNow, - endpointState)); + endpointState, + redundancy, + standbyProjection)); /// Test-only Props exposing the deferred-write bound and the plan-step seams. internal static Props PropsForTests( @@ -461,10 +470,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers bool dbLess = false, IDataPlaneAuthoritySink? dataPlaneAuthority = null, Func? utcNow = null, - IOpcUaEndpointState? endpointState = null) + IOpcUaEndpointState? endpointState = null, + WarmRedundancySettings? redundancy = null, + IStandbyProjection? standbyProjection = null) : this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow, probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null, - endpointState) + endpointState, redundancy, standbyProjection) { } @@ -488,8 +499,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers Func? utcNow, int deferredWriteCapacity, TestHooks? hooks, - IOpcUaEndpointState? endpointState) + IOpcUaEndpointState? endpointState, + WarmRedundancySettings? redundancy = null, + IStandbyProjection? standbyProjection = null) { + _redundancy = redundancy ?? WarmRedundancySettings.Unset; + _standbyProjection = standbyProjection; _deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity)); _hooks = hooks; _sink = sink; @@ -523,6 +538,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers Receive(_ => OnHealthTick()); Receive(HandlePeerProbe); Receive(_ => { /* PubSub ack */ }); + // Gitea #717 spec §4.1 / owner answer Q2: under Warm the verdict re-derives on membership and reachability, not + // only on snapshots. Nothing subscribes these under Unset/None. + Receive(_ => RecomputeServiceLevel()); + Receive(_ => RecomputeServiceLevel()); + Receive(_ => RecomputeServiceLevel()); + Receive(_ => RecomputeServiceLevel()); } /// @@ -543,6 +564,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers base.PreRestart(reason, message); } + /// + protected override void PostStop() + { + if (_redundancy.IsWarm) + { + try { _cluster.Unsubscribe(Self); } + catch (Exception) { /* the cluster is shutting down with us */ } + } + + base.PostStop(); + } + /// protected override void PreStart() { @@ -551,6 +584,20 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self)); } + if (_redundancy.IsWarm) + { + OtOpcUaTelemetry.SetWarmRole(0); + try + { + _cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents, + typeof(ClusterEvent.IMemberEvent), typeof(ClusterEvent.IReachabilityEvent)); + } + catch (Exception ex) + { + _log.Warning(ex, "OpcUaPublish: Warm redundancy could not subscribe to cluster membership; the verdict re-derives on snapshots only"); + } + } + // DB-backed nodes refresh their cached probe status; DB-less nodes still need periodic // snapshot/probe-verdict age checks even when no new messages arrive. Legacy nodes keep // their role-only behavior without a timer. @@ -672,7 +719,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers } } - private void HandleRebuild(RebuildAddressSpace msg) => StartRebuild(msg, Sender); + private void HandleRebuild(RebuildAddressSpace msg) + { + if (msg.Artifact is { Length: > 0 } artifact) CheckRedundancyMode(artifact); + StartRebuild(msg, Sender); + } /// /// The entry half of a rebuild, on the actor thread. The raw-sink fallback (no applier) and the @@ -1465,26 +1516,34 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers } } - private void HandleServiceLevelChanged(ServiceLevelChanged msg) + private void HandleServiceLevelChanged(ServiceLevelChanged msg) => TryPublishServiceLevel(msg.ServiceLevel); + + /// The dedup/publish/metric step behind ; synchronous so a Warm + /// transition can order it against the projection steps (spec §4.7). + /// The level to publish. + /// True when the level is published (or already was). + private bool TryPublishServiceLevel(byte level) { // Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a // node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's // built-in default (255 = full service) standing — a degraded node wrongly advertising 255. - if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return; + if (_publishedAtLeastOnce && level == _lastServiceLevel) return true; try { - _serviceLevel.Publish(msg.ServiceLevel); - _lastServiceLevel = msg.ServiceLevel; + _serviceLevel.Publish(level); + _lastServiceLevel = level; _publishedAtLeastOnce = true; OtOpcUaTelemetry.ServiceLevelChange.Add(1, - new KeyValuePair("level", msg.ServiceLevel)); - _log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel); + new KeyValuePair("level", level)); + _log.Debug("OpcUaPublish: ServiceLevel={Level}", level); + return true; } catch (Exception ex) { // A failed write may be partially applied; even a previous successful value must retry. _publishedAtLeastOnce = false; - _log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", msg.ServiceLevel); + _log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", level); + return false; } } @@ -1494,6 +1553,28 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers /// wired) or via the legacy role-only seam (back-compat / bootstrap). private void HandleRedundancyStateChanged(RedundancyStateChanged msg) { + if (_redundancy.IsWarm) + { + // Gitea #717 spec §4.2: under Warm a snapshot older than the last accepted one (a late heartbeat from an old + // singleton incarnation) is dropped outright. An unrevisioned one (a pre-#717 publisher mid-upgrade) still + // feeds the health inputs but can never make this node Active. Unset/None keep today's unconditional + // replace below. + switch (_revisionGate.Offer(msg)) + { + case SnapshotRejection.OlderRevision: + OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair("reason", "older_revision")); + _log.Debug("OpcUaPublish: dropped redundancy snapshot {Revision} (last accepted {Last})", + msg.Revision, _revisionGate.LastAccepted); + return; + case SnapshotRejection.Unrevisioned: + OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair("reason", "unrevisioned")); + break; + default: + _acceptedSnapshot = msg; + break; + } + } + _lastSnapshot = msg; RecomputeServiceLevel(); } @@ -1576,6 +1657,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers private void RecomputeServiceLevel() { if (_localNode is null || _lastSnapshot is null) return; + if (_redundancy.IsWarm) + { + RecomputeWarm(); + return; + } var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value); @@ -1635,6 +1721,197 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs))); } + // ---------------------------------------------------------------- Warm redundancy (Gitea #717) + + private sealed class WarmRetry { public static readonly WarmRetry Instance = new(); private WarmRetry() { } } + + private readonly WarmRedundancySettings _redundancy; + private readonly IStandbyProjection? _standbyProjection; + private readonly SnapshotRevisionGate _revisionGate = new(); + /// The latest ACCEPTED revisioned snapshot — the only one the Warm verdict reads (spec §4.2). + private RedundancyStateChanged? _acceptedSnapshot; + /// The role the data plane is in. A Warm node manager boots Standby, so this starts Standby. + private WarmRole _warmRole = WarmRole.Standby; + /// A promotion whose Active ServiceLevel publish failed (spec §4.7): the data plane is Active and the + /// level is retried; the Standby level stays visible meanwhile, never the reverse. + private bool _promotionLevelPending; + private DateTime? _disagreeSinceUtc; + private bool _disagreementWarned; + + /// How long membership and snapshot may disagree before a Warning (two heartbeats, spec §4.1). + internal static readonly TimeSpan WarmDisagreementWarning = TimeSpan.FromSeconds(20); + + /// The current Warm role (test observation). + internal WarmRole CurrentWarmRole => _warmRole; + + /// A promoted node whose Active ServiceLevel publish is being retried (test observation). + internal bool PromotionLevelPending => _promotionLevelPending; + + /// + /// The Warm ServiceLevel + role path (spec §4.1, §4.3, §4.7): the health facts the non-Warm paths use, the + /// Warm verdict, then a transition in the fixed step order when the verdict changed, else just the band. + /// + private void RecomputeWarm() + { + var self = _localNode!.Value; + var entry = _lastSnapshot!.Nodes.FirstOrDefault(n => n.NodeId == self); + var now = _utcNow(); + var memberState = SafeSelfStatus(); + var listening = EndpointListening(); + + bool dbReachable; + bool stale; + if (_dbLess || _dbHealthProbe is null || _lastDbHealth is null) + { + dbReachable = true; + stale = entry is null || (now - entry.AsOfUtc) > _staleWindow; + } + else + { + dbReachable = _lastDbHealth.Reachable; + stale = !_lastDbHealth.Reachable + || (now - _lastDbHealth.AsOfUtc) > _staleWindow + || entry is null || (now - entry.AsOfUtc) > _staleWindow; + } + + var verdict = EvaluateWarmVerdict(self, memberState, listening); + TrackDisagreement(verdict, now); + + var detached = entry is null || entry.Role == RedundancyRole.Detached; + var inputs = new NodeHealthInputs(memberState, dbReachable, OpcUaProbeOk(), stale, IsDriverPrimary: false, listening); + var target = detached ? WarmRole.Standby : verdict.Role; + var level = detached ? (byte)0 : ServiceLevelCalculator.ComputeWarm(inputs, target == WarmRole.Active); + ApplyWarmTransition(target, level, verdict.Reason); + } + + private WarmVerdict EvaluateWarmVerdict(NodeId self, MemberStatus memberState, bool listening) + { + NodeId? oldest = null, oldestReachable = null; + var primaryUnreachable = false; + var snapshotPrimary = _acceptedSnapshot?.Nodes.FirstOrDefault(n => n.Role == RedundancyRole.Primary)?.NodeId; + try + { + var state = _cluster.State; + var unreachable = state.Unreachable.ToHashSet(); + oldest = WarmRoleAuthority.OldestUpMember(state.Members, "driver"); + oldestReachable = WarmRoleAuthority.OldestUpMember(state.Members, "driver", unreachable); + primaryUnreachable = snapshotPrimary is { } sp && unreachable.Any(m => WarmRoleAuthority.ToNodeId(m) == sp); + } + catch (Exception ex) + { + _log.Debug(ex, "OpcUaPublish: cluster state unavailable for the Warm verdict; treating membership as unknown"); + } + + return WarmRoleAuthority.Evaluate(new WarmRoleInputs( + self, memberState == MemberStatus.Up, listening, oldest, oldestReachable, + HasAcceptedSnapshot: _acceptedSnapshot is not null, snapshotPrimary, primaryUnreachable, + _redundancy.PromoteWhilePrimaryUnreachable)); + } + + private void TrackDisagreement(WarmVerdict verdict, DateTime now) + { + if (!verdict.Disagreement) + { + _disagreeSinceUtc = null; + _disagreementWarned = false; + return; + } + + _disagreeSinceUtc ??= now; + if (_disagreementWarned || now - _disagreeSinceUtc.Value <= WarmDisagreementWarning) return; + _disagreementWarned = true; + _log.Warning( + "OpcUaPublish: Warm redundancy — cluster membership and the redundancy snapshot (revision {Revision}) have disagreed about this node being Primary for more than {Seconds}s; the node stays {Role}", + _revisionGate.LastAccepted, WarmDisagreementWarning.TotalSeconds, _warmRole); + } + + /// + /// Moves the data plane to in the spec §4.7 order and publishes + /// . Invariant: an Active-band ServiceLevel is never visible while the projection is + /// closed — a promotion publishes the level LAST and, if that publish fails, retries it; a demotion publishes + /// the Standby level FIRST and degrades the data whether or not that publish succeeded (failing safe). + /// + private void ApplyWarmTransition(WarmRole target, byte level, string reason) + { + if (target == _warmRole) + { + if (!TryPublishServiceLevel(level) && target == WarmRole.Active) ScheduleWarmRetry(); + else if (target == WarmRole.Active) _promotionLevelPending = false; + return; + } + + var projection = _standbyProjection; + if (target == WarmRole.Active) + { + projection?.SetDataProjection(false); // 1. open the read projection + projection?.QueueCurrentToLiveItems(false); // 2. current Good value to every live data item + projection?.SetStandbyDelivery(false); // 3. first deliveries decided Active + projection?.SetAlarmSuppression(false); // 4. new alarm events delivered + projection?.RefreshAllEventItems(); // 5. promotion refresh on every live event item + _warmRole = WarmRole.Active; + if (TryPublishServiceLevel(level)) // 6. Active ServiceLevel, last + { + _promotionLevelPending = false; + } + else + { + _promotionLevelPending = true; + ScheduleWarmRetry(); + } + } + else + { + TryPublishServiceLevel(level); // 1. Standby ServiceLevel, first (continue on failure) + projection?.SetStandbyDelivery(true); // 2. the D4 boundary + projection?.SetAlarmSuppression(true); // 3. suppress new alarm events + projection?.SetDataProjection(true); // 4. close the read projection + projection?.QueueCurrentToLiveItems(true); // 5. BadOutOfService to every live data item + _warmRole = WarmRole.Standby; + _promotionLevelPending = false; + } + + OtOpcUaTelemetry.SetWarmRole(_warmRole == WarmRole.Active ? 1 : 0); + OtOpcUaTelemetry.WarmTransition.Add(1, + new KeyValuePair("direction", target == WarmRole.Active ? "promote" : "demote"), + new KeyValuePair("reason", reason)); + _log.Info( + "OpcUaPublish: Warm redundancy {Direction} to {Role} (reason={Reason}, revision={Revision}, ServiceLevel={Level})", + target == WarmRole.Active ? "promotion" : "demotion", _warmRole, reason, _revisionGate.LastAccepted, level); + } + + /// + /// Gitea #717 spec §3 deploy-time cross-check: under None or Warm, an artifact whose cluster + /// RedundancyMode disagrees with the local mode (Hot included, owner answer Q5) logs an Error and + /// counts otopcua.redundancy.mode_mismatch. The node keeps its local mode — the only value available + /// before SDK start. Unset has opted out and is not checked. + /// + /// The applied artifact bytes. + private void CheckRedundancyMode(byte[] artifact) + { + if (_redundancy.Mode == OpcUaRedundancyMode.Unset || _localNode is null) return; + string? clusterMode; + try + { + clusterMode = DeploymentArtifact.ReadClusterRedundancyMode(artifact, _localNode.Value.Value); + } + catch (Exception ex) + { + _log.Debug(ex, "OpcUaPublish: could not read the artifact's cluster RedundancyMode"); + return; + } + + if (clusterMode is null) return; + if (string.Equals(clusterMode, _redundancy.Mode.ToString(), StringComparison.OrdinalIgnoreCase)) return; + OtOpcUaTelemetry.RedundancyModeMismatch.Add(1, + new KeyValuePair("local", _redundancy.Mode.ToString()), + new KeyValuePair("cluster", clusterMode)); + _log.Error( + "OpcUaPublish: the deployed ServerCluster.RedundancyMode for this node's cluster is {ClusterMode} but OpcUa:Redundancy:Mode is {LocalMode}; the node keeps running as {LocalMode} (local configuration is the authority — fix whichever is wrong)", + clusterMode, _redundancy.Mode, _redundancy.Mode); + } + + private void ScheduleWarmRetry() => Timers.StartSingleTimer("warm-retry", WarmRetry.Instance, TimeSpan.FromSeconds(1)); + /// The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200, /// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam. private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/WarmRoleAuthority.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/WarmRoleAuthority.cs new file mode 100644 index 000000000..6a96fbaaf --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/WarmRoleAuthority.cs @@ -0,0 +1,109 @@ +using Akka.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Types; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; + +/// The Warm role (Gitea #717 spec §4.1). +public enum WarmRole +{ + /// Degraded standby: projected data, suppressed new alarm events, Degraded-band ServiceLevel. + Standby, + + /// Serves data and alarms; Healthy-band ServiceLevel. + Active, +} + +/// The facts the Warm verdict is a pure function of. +/// This node's canonical host:port id. +/// This node's own member status is Up. +/// This node's OPC UA endpoint did not fail to start (#762). +/// The oldest Up driver member in this node's OWN cluster state (the singleton +/// placement rule), or null when there is none. +/// The same, ignoring members this node sees as unreachable. +/// A revisioned snapshot has been accepted (spec §4.2). Unrevisioned snapshots +/// never count. +/// The Primary named by the latest accepted revisioned snapshot, or null. +/// This node sees as unreachable. +/// The owner-answer-Q2 option (default on). +public readonly record struct WarmRoleInputs( + NodeId Self, + bool SelfUp, + bool EndpointListening, + NodeId? OldestUpDriver, + NodeId? OldestReachableUpDriver, + bool HasAcceptedSnapshot, + NodeId? SnapshotPrimary, + bool SnapshotPrimaryUnreachable, + bool PromoteWhilePrimaryUnreachable); + +/// The verdict and why. +/// Active or Standby. +/// A short reason for the transition metric/log: boot, endpoint, +/// membership, snapshot, unreachable-primary or agreed. +/// Membership (2) and snapshot (3) disagree about whether this node is Primary. +public readonly record struct WarmVerdict(WarmRole Role, string Reason, bool Disagreement); + +/// +/// The Warm role verdict (Gitea #717 spec §4.1), a pure function so every row is unit-testable. Active iff the +/// node is Up and listening, its own cluster state names it the oldest Up driver member, and the latest accepted +/// revisioned snapshot names it Primary. Anything else — no snapshot yet, an unrevisioned snapshot, a +/// disagreement — is Standby. With the survivor of a +/// crashed Primary goes Active once that Primary is unreachable, before auto-down removes it. +/// +/// +/// This is a separate output from PrimaryGatePolicy, whose lone-node allow is unchanged for writes and +/// acks (D7). +/// +public static class WarmRoleAuthority +{ + /// Evaluates the verdict. + /// The inputs. + /// The verdict. + public static WarmVerdict Evaluate(in WarmRoleInputs i) + { + if (!i.SelfUp) return new(WarmRole.Standby, "membership", false); + if (!i.EndpointListening) return new(WarmRole.Standby, "endpoint", false); + if (!i.HasAcceptedSnapshot) return new(WarmRole.Standby, "boot", false); + + var membershipSaysSelf = i.OldestUpDriver == i.Self; + var snapshotSaysSelf = i.SnapshotPrimary == i.Self; + var disagreement = membershipSaysSelf != snapshotSaysSelf; + + if (membershipSaysSelf && snapshotSaysSelf) return new(WarmRole.Active, "agreed", false); + + // Owner answer Q2: a SIGKILLed Primary stays an Up-but-unreachable member until auto-down removes it, and the + // singleton usually died with it. Evaluate (2) and (3) with unreachable members excluded. + if (i.PromoteWhilePrimaryUnreachable + && i.OldestReachableUpDriver == i.Self + && (snapshotSaysSelf || (i.SnapshotPrimary is not null && i.SnapshotPrimaryUnreachable))) + { + return new(WarmRole.Active, "unreachable-primary", disagreement); + } + + return new(WarmRole.Standby, membershipSaysSelf ? "snapshot" : "membership", disagreement); + } + + /// The oldest Up member carrying (the singleton placement rule), optionally + /// ignoring members. + /// The current members. + /// The role. + /// Members to ignore, or null. + /// Its canonical node id, or null. + public static NodeId? OldestUpMember(IEnumerable members, string role, IReadOnlySet? unreachable = null) + { + Member? oldest = null; + foreach (var m in members) + { + if (m.Status != MemberStatus.Up || !m.HasRole(role)) continue; + if (unreachable is not null && unreachable.Contains(m)) continue; + if (oldest is null || m.IsOlderThan(oldest)) oldest = m; + } + + return oldest is null ? null : ToNodeId(oldest); + } + + /// The canonical host:port id of a member (the RedundancyStateActor format). + /// The member. + /// The node id. + public static NodeId ToNodeId(Member m) => NodeId.Parse($"{m.Address.Host ?? string.Empty}:{m.Address.Port ?? 0}"); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index 8acc830be..54cb60809 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -566,6 +566,9 @@ public static class ServiceCollectionExtensions // Optional: a harness that builds this graph without the hosted service registers none, and both // consumers then behave as if the endpoint were listening (the pre-#762 behaviour). var endpointState = resolver.GetService(); + // Gitea #717: OpcUa:Redundancy (registered by the Host from the validated OpcUa options). Absent in harnesses ⇒ + // Unset ⇒ today's behaviour. The standby projection is late-bound like the ServiceLevel publisher. + var redundancy = resolver.GetService() ?? WarmRedundancySettings.Unset; var publishActor = system.ActorOf( OpcUaPublishActor.Props( sink: addressSpaceSink, @@ -576,7 +579,9 @@ public static class ServiceCollectionExtensions dbHealthProbe: dbHealth, dbLess: dbFactory is null, dataPlaneAuthority: dataPlaneAuthority, - endpointState: endpointState), + endpointState: endpointState, + redundancy: redundancy, + standbyProjection: redundancy.IsWarm ? resolver.GetService() : null), OpcUaPublishActorName); registry.Register(publishActor); @@ -670,7 +675,9 @@ public static class ServiceCollectionExtensions // lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under // the InProcess default, when the seam is not listed, and in test graphs that register no provider. scriptedAlarmRunner: resolver.GetService() - ?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)), + ?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm), + // Gitea #717 spec §4.2: drop older redundancy snapshots under Warm only. + enforceSnapshotOrdering: redundancy.IsWarm), DriverHostActorName); registry.Register(driverHost); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/WarmRedundancyRevisionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/WarmRedundancyRevisionTests.cs new file mode 100644 index 000000000..b80af2c5a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/WarmRedundancyRevisionTests.cs @@ -0,0 +1,154 @@ +using Akka.Actor; +using Akka.Cluster; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Gitea #717 spec §4.2 (ordered snapshots) and §4.3 (Warm bands): the revision the singleton stamps, its order, +/// the subscriber gate, the wire shape of the extended message, and the Warm ServiceLevel truth table. +/// +public sealed class WarmRedundancyRevisionTests : ControlPlaneActorTestBase +{ + [Fact] + public void The_singleton_stamps_an_increasing_revision_on_every_publish_heartbeats_included() + { + var probe = CreateTestProbe(); + Sys.ActorOf(RedundancyStateActor.Props(msg => probe.Ref.Tell(msg), TimeSpan.FromMilliseconds(200))); + var first = probe.ExpectMsg(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); + var second = probe.ExpectMsg(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); + + first.Revision.ShouldNotBeNull(); + second.Revision.ShouldNotBeNull(); + second.Revision!.CompareTo(first.Revision).ShouldBeGreaterThan(0); + second.Revision.Sequence.ShouldBe(first.Revision!.Sequence + 1); + first.Revision.PublisherUpNumber.ShouldBeGreaterThan(0L, "the publisher's Akka up-number"); + } + + /// Akka.NET keeps Member.UpNumber internal; an upgrade that renames it must fail here, not degrade + /// every revision to up-number 0 silently. + [Fact] + public void Akka_member_up_number_is_readable() + { + AwaitCondition(() => Akka.Cluster.Cluster.Get(Sys).SelfMember.Status == MemberStatus.Up, TimeSpan.FromSeconds(10), + cancellationToken: TestContext.Current.CancellationToken); + RedundancyStateActor.ReadUpNumber(Akka.Cluster.Cluster.Get(Sys).SelfMember).ShouldNotBeNull().ShouldBeGreaterThan(0); + } + + [Fact] + public void A_singleton_restarted_in_the_same_process_outranks_its_predecessor() + { + var a = RedundancyStateActor.NextIncarnationTick(); + var b = RedundancyStateActor.NextIncarnationTick(); + b.ShouldBeGreaterThan(a, "strictly increasing even within one tick"); + new SnapshotRevision(1, b, 1).CompareTo(new SnapshotRevision(1, a, 500)).ShouldBeGreaterThan(0); + } + + [Fact] + public void Revisions_order_by_up_number_then_incarnation_then_sequence() + { + var baseline = new SnapshotRevision(2, 100, 50); + new SnapshotRevision(3, 1, 1).CompareTo(baseline).ShouldBeGreaterThan(0, "a handover to a younger member outranks"); + new SnapshotRevision(1, 999, 999).CompareTo(baseline).ShouldBeLessThan(0, "a departed older member's late publish loses"); + new SnapshotRevision(2, 101, 1).CompareTo(baseline).ShouldBeGreaterThan(0); + new SnapshotRevision(2, 100, 51).CompareTo(baseline).ShouldBeGreaterThan(0); + new SnapshotRevision(2, 100, 50).CompareTo(baseline).ShouldBe(0); + } + + [Fact] + public void The_gate_accepts_only_newer_revisions_and_never_advances_on_an_unrevisioned_snapshot() + { + var gate = new SnapshotRevisionGate(); + gate.Offer(Snap(null)).ShouldBe(SnapshotRejection.Unrevisioned); + gate.LastAccepted.ShouldBeNull(); + gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.None); + gate.Offer(Snap(new(1, 9, 9))).ShouldBe(SnapshotRejection.OlderRevision, "an old incarnation's late heartbeat"); + gate.Offer(Snap(new(2, 1, 5))).ShouldBe(SnapshotRejection.OlderRevision, "a duplicate"); + gate.Offer(Snap(new(2, 1, 6))).ShouldBe(SnapshotRejection.None); + gate.LastAccepted.ShouldBe(new SnapshotRevision(2, 1, 6)); + } + + [Fact] + public void The_extended_message_round_trips_and_a_pre_717_payload_decodes_with_no_revision() + { + var msg = Snap(new SnapshotRevision(3, 42, 7)); + var serializer = Sys.Serialization.FindSerializerFor(msg); + var manifest = Akka.Serialization.Serialization.ManifestFor(serializer, msg); + var round = (RedundancyStateChanged)Sys.Serialization.Deserialize(serializer.ToBinary(msg), serializer.Identifier, manifest); + round.Revision.ShouldBe(msg.Revision); + round.Nodes.Single().NodeId.ShouldBe(msg.Nodes.Single().NodeId); + + // A pre-#717 publisher's payload: the same message without the field. + var legacy = Snap(null); + var legacyBytes = serializer.ToBinary(legacy); + var text = System.Text.Encoding.UTF8.GetString(legacyBytes); + var stripped = System.Text.Encoding.UTF8.GetBytes(text.Replace(",\"Revision\":null", string.Empty, StringComparison.Ordinal)); + var decoded = (RedundancyStateChanged)Sys.Serialization.Deserialize(stripped, serializer.Identifier, manifest); + decoded.Revision.ShouldBeNull(); + decoded.Nodes.Count.ShouldBe(1); + } + + private static RedundancyStateChanged Snap(SnapshotRevision? revision) => + new(new[] { new NodeRedundancyState(NodeId.Parse("a:1"), RedundancyRole.Primary, true, true, DateTime.UtcNow) }, + CorrelationId.NewId(), revision); +} + +/// Gitea #717 spec §4.3: the Warm ServiceLevel bands (pure; no actor system). +public sealed class WarmServiceLevelBandTests +{ + + public static IEnumerable AllInputs() + { + foreach (var member in new[] { MemberStatus.Up, MemberStatus.Joining, MemberStatus.Leaving, MemberStatus.Down }) + foreach (var db in new[] { true, false }) + foreach (var probe in new[] { true, false }) + foreach (var stale in new[] { true, false }) + foreach (var listening in new[] { true, false }) + yield return [member, db, probe, stale, listening]; + } + + [Theory] + [MemberData(nameof(AllInputs))] + public void Every_standby_level_is_below_every_active_level_that_serves_data( + MemberStatus member, bool db, bool probe, bool stale, bool listening) + { + var h = new NodeHealthInputs(member, db, probe, stale, IsDriverPrimary: false, listening); + var standby = ServiceLevelCalculator.ComputeWarm(h, isActive: false); + var active = ServiceLevelCalculator.ComputeWarm(h, isActive: true); + standby.ShouldBeLessThanOrEqualTo(ServiceLevelCalculator.WarmStandbyHealthy); + if (active > 0) active.ShouldBeGreaterThanOrEqualTo(ServiceLevelCalculator.WarmActiveCritical); + if (active > 0) standby.ShouldBeLessThan(active); + } + + [Theory] + [InlineData(true, true, false, true, 250)] + [InlineData(true, true, true, true, 200)] + [InlineData(false, true, true, true, 100)] + [InlineData(false, true, false, true, 100)] + [InlineData(true, false, false, true, 100)] + [InlineData(true, true, false, false, 90)] + [InlineData(true, true, true, false, 50)] + [InlineData(false, true, false, false, 50)] + [InlineData(true, false, false, false, 50)] + public void Warm_bands_match_the_spec_table(bool db, bool probe, bool stale, bool active, int expected) + { + ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, db, probe, stale, IsDriverPrimary: false), active) + .ShouldBe((byte)expected); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Down_or_a_failed_endpoint_is_zero_in_either_role(bool active) + { + ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Down, true, true, false, false), active).ShouldBe((byte)0); + ServiceLevelCalculator.ComputeWarm(new NodeHealthInputs(MemberStatus.Up, true, true, false, false, EndpointListening: false), active) + .ShouldBe((byte)0); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/OpcUaApplicationHostOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/OpcUaApplicationHostOptionsValidatorTests.cs index 87ad534b7..5dd4fa90f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/OpcUaApplicationHostOptionsValidatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/OpcUaApplicationHostOptionsValidatorTests.cs @@ -132,3 +132,38 @@ public sealed class OpcUaApplicationHostOptionsValidatorTests if (!valid) result.Failures!.ShouldContain(f => f.Contains("OpcUa:Discovery")); } } + +/// Gitea #717 spec §3: the redundancy rules the startup validator and the driver-role guard enforce. +public sealed class OpcUaRedundancyValidationTests +{ + private static readonly OpcUaApplicationHostOptionsValidator Sut = new(); + + [Theory] + [InlineData("Unset", new string[0], true)] + [InlineData("None", new string[0], true)] + [InlineData("None", new[] { "urn:peer" }, false)] + [InlineData("Warm", new[] { "urn:peer" }, true)] + [InlineData("Warm", new string[0], false)] + [InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)] + [InlineData("Warm", new[] { "urn:OtOpcUa" }, false)] + [InlineData("Hot", new[] { "urn:peer" }, false)] + [InlineData("Bogus", new string[0], false)] + public void Redundancy_matrix(string mode, string[] peers, bool valid) + { + var result = Sut.Validate(null, new OpcUaApplicationHostOptions + { + Redundancy = new OpcUaRedundancyOptions { Mode = mode }, + PeerApplicationUris = peers.ToList(), + }); + result.Succeeded.ShouldBe(valid, result.FailureMessage); + } + + [Fact] + public void Warm_on_a_node_without_the_driver_role_fails_start() + { + Should.Throw(() => OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: false)); + OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("Warm", hasDriver: true); + OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(null, hasDriver: false); + OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm("None", hasDriver: false); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs index 4f1a82ef9..725fe1376 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs @@ -44,6 +44,9 @@ internal sealed class ServerEventRig : IAsyncDisposable public string EndpointUrl => $"opc.tcp://127.0.0.1:{_port}/OtOpcUa"; + /// The configured OpcUa:ApplicationUri (the effective one: the rig loads no XML). + public string ApplicationUri => _hostOptions.ApplicationUri; + public OtOpcUaNodeManager Nm => _server.NodeManager!; internal OtOpcUaSdkServer Server => _server; @@ -57,7 +60,12 @@ internal sealed class ServerEventRig : IAsyncDisposable public SdkAddressSpaceSink Sink { get; private set; } = null!; - public static async Task StartAsync(string name, bool encrypted, CancellationToken ct) + public static Task StartAsync(string name, bool encrypted, CancellationToken ct) => + StartAsync(name, encrypted, ct, configure: null); + + /// Starts the rig, letting the caller adjust the host options first (Gitea #717: redundancy mode, peers). + public static async Task StartAsync( + string name, bool encrypted, CancellationToken ct, Action? configure) { var rig = new ServerEventRig(Path.Combine(Path.GetTempPath(), $"otopcua-{name}-{Guid.NewGuid():N}")); rig._hostOptions = new OpcUaApplicationHostOptions @@ -71,6 +79,7 @@ internal sealed class ServerEventRig : IAsyncDisposable : [OpcUaSecurityProfile.None], AutoAcceptUntrustedClientCertificates = true, }; + configure?.Invoke(rig._hostOptions); (rig._host, rig._server) = await OpcUaHostStart.StartAsync( port => { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs new file mode 100644 index 000000000..6f3afe48f --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs @@ -0,0 +1,616 @@ +using Opc.Ua; +using Opc.Ua.Client; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; + +/// +/// Gitea #717 over the wire, against the real and node manager: the redundancy +/// metadata per mode (T2), the Standby read projection and write refusal (T6), the D4 first-delivery decision record +/// across the three demotion timings T0 established (T7), the live-item queue on promotion/demotion (T8), and the +/// alarm suppression, refusal and promotion refresh (T9). Transitions are driven through the server's own +/// in the spec §4.7 step order; the actor that owns that order has its own tests. +/// +[Trait("Category", "Integration")] +public sealed class WarmRedundancyWireTests +{ + private const string Peer = "urn:OtOpcUa:peer-717"; + + private const NodePermissions All = + NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead + | NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve + | NodePermissions.MethodCall | NodePermissions.WriteOperate; + + private static Task StartModeAsync(string name, string mode, CancellationToken ct) => + ServerEventRig.StartAsync(name, encrypted: false, ct, o => + { + o.Redundancy = new OpcUaRedundancyOptions { Mode = mode }; + if (mode == "Warm") o.PeerApplicationUris = [Peer]; + }); + + /// Spec §4.7 promotion, in order (the actor's sequence, minus the ServiceLevel publish). + internal static void Promote(IStandbyProjection p) + { + p.SetDataProjection(false); + p.QueueCurrentToLiveItems(false); + p.SetStandbyDelivery(false); + p.SetAlarmSuppression(false); + p.RefreshAllEventItems(); + } + + /// Spec §4.7 demotion, in order (after the Standby ServiceLevel publish). + internal static void Demote(IStandbyProjection p) + { + p.SetStandbyDelivery(true); + p.SetAlarmSuppression(true); + p.SetDataProjection(true); + p.QueueCurrentToLiveItems(true); + } + + private static CancellationToken Deadline(int seconds = 90) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(seconds)); + return cts.Token; + } + + private static RequestHeader H() => new() { Timestamp = DateTimeUtc.Now, TimeoutHint = 10_000 }; + + private static NodeId Raw(ISession session, string id) => + new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri)); + + private static async Task ReadAsync(ISession session, NodeId node, CancellationToken ct, uint attribute = Attributes.Value, + string? indexRange = null) + { + var r = await ((SessionClient)session).ReadAsync(H(), 0, TimestampsToReturn.Both, + new[] { new ReadValueId { NodeId = node, AttributeId = attribute, IndexRange = indexRange } }, ct); + return r.Results[0]; + } + + // ------------------------------------------------------------------ T2 metadata + + [Fact] + public async Task Warm_advertises_non_transparent_metadata_self_first_and_boots_at_ServiceLevel_1() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717Meta", "Warm", ct); + using var session = await rig.ConnectAsync(anonymous: true, ct); + + Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue()) + .ShouldBe((int)RedundancySupport.Warm); + var uris = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct); + uris.StatusCode.ShouldBe(StatusCodes.Good); + ((string[])uris.ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer }); + ((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer }); + Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.WarmBootServiceLevel); + + var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[] + { + new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward, + ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All }, + }, ct); + browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.NonTransparentRedundancyType)); + + var write = await ((SessionClient)session).WriteAsync(H(), new[] + { + new WriteValue { NodeId = VariableIds.Server_ServerRedundancy_ServerUriArray, AttributeId = Attributes.Value, + Value = new DataValue(new Variant(new[] { "x" })) }, + }, ct); + StatusCode.IsBad(write.Results[0]).ShouldBeTrue($"ServerUriArray is read-only, got {write.Results[0]}"); + } + + [Fact] + public async Task None_advertises_RedundancySupport_None_and_ServerUriArray_is_absent_even_by_NodeId() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717None", "None", ct); + using var session = await rig.ConnectAsync(anonymous: true, ct); + + Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue()) + .ShouldBe((int)RedundancySupport.None); + (await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct)).StatusCode + .ShouldBe(StatusCodes.BadNodeIdUnknown, "T0a: the SDK's orphan i=11314 is deleted under None"); + var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[] + { + new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward, + ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All }, + }, ct); + browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.ServerRedundancyType)); + ((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri }); + } + + /// The default-off pin: an Unset server is today's server — SDK-default RedundancySupport, the SDK's own + /// (orphan) i=11314, ServerRedundancyType, the #885 boot level 0, Good data with no projection, and no Warm hooks. + [Fact] + public async Task Unset_is_todays_server_exactly() + { + var ct = Deadline(); + await using var rig = await ServerEventRig.StartAsync("Warm717Unset", encrypted: false, ct); + rig.Server.RedundancyMode.ShouldBe(OpcUaRedundancyMode.Unset); + rig.Server.CreateStandbyProjection().ShouldBeNull(); + rig.Nm.Standby.WarmMode.ShouldBeFalse(); + var id = rig.EnsureRawVariable("u", "Double", writable: false); + rig.Sink.WriteValue(id, 4.5, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + var support = await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct); + support.StatusCode.ShouldBe(StatusCodes.Good); + Convert.ToInt32(support.ClrValue()).ShouldBe((int)RedundancySupport.None, "the SDK default, never written by us"); + var orphan = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct); + orphan.StatusCode.ShouldBe(StatusCodes.Good, "Unset leaves the SDK's orphan i=11314 exactly as the SDK builds it"); + Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel); + var value = await ReadAsync(session, Raw(session, id), ct); + value.StatusCode.ShouldBe(StatusCodes.Good); + value.ClrValue().ShouldBe(4.5); + } + + // ------------------------------------------------------------------ T6 read projection + write refusal + + [Fact] + public async Task Standby_projects_ordinary_tag_reads_but_not_metadata_and_promotion_reads_the_latest_stored_value() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717Read", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection().ShouldNotBeNull(); + var raw = rig.EnsureRawVariable("t1", "Double", writable: false); + rig.Sink.EnsureVariable("EqA/t1ref", null, "t1ref", "Double", writable: false, AddressSpaceRealm.Uns); + var arr = $"{RawDeviceFolder}/arr"; + rig.Sink.EnsureVariable(arr, RawDeviceFolder, "arr", "Int32", writable: false, AddressSpaceRealm.Raw, isArray: true); + rig.EnsureEquipment(); + rig.Sink.EnsureVariable("EqA/vt", "EqA", "vt", "Double", writable: false, AddressSpaceRealm.Uns); + rig.MaterialiseScripted(); + var ts = DateTime.UtcNow; + rig.Sink.WriteValue(raw, 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw); + rig.Sink.WriteValue("EqA/t1ref", 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns); + rig.Sink.WriteValue("EqA/vt", 2.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns); + rig.Sink.WriteValue(arr, new[] { 1, 2, 3 }, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + foreach (var node in new[] { Raw(session, raw), UnsNodeId(session, "EqA/t1ref"), UnsNodeId(session, "EqA/vt") }) + { + var dv = await ReadAsync(session, node, ct); + dv.StatusCode.ShouldBe(StatusCodes.BadOutOfService, node.ToString()); + dv.WrappedValue.IsNull.ShouldBeTrue(); + (dv.ServerTimestamp != DateTimeUtc.MinValue).ShouldBeTrue(); + } + + (await ReadAsync(session, Raw(session, arr), ct, indexRange: "1")).StatusCode.ShouldBe(StatusCodes.BadOutOfService); + (await ReadAsync(session, Raw(session, raw), ct, Attributes.DisplayName)).StatusCode.ShouldBe(StatusCodes.Good); + (await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).StatusCode.ShouldBe(StatusCodes.Good); + (await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).StatusCode.ShouldBe(StatusCodes.Good); + var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct); + (await ReadAsync(session, severity, ct)).StatusCode.ShouldBe(StatusCodes.Good, "a condition child is not an ordinary tag"); + + // Acquisition continues while projected; promotion reads the latest stored value with no new driver publish. + rig.Sink.WriteValue(raw, 9.25, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + Promote(projection); + var after = await ReadAsync(session, Raw(session, raw), ct); + after.StatusCode.ShouldBe(StatusCodes.Good); + after.ClrValue().ShouldBe(9.25); + var ranged = await ReadAsync(session, Raw(session, arr), ct, indexRange: "1"); + ranged.StatusCode.ShouldBe(StatusCodes.Good); + ((int[])ranged.ClrValue()!).ShouldBe(new[] { 2 }, "an IndexRange read reaches the stored array once promoted"); + } + + [Fact] + public async Task Standby_projection_never_leaks_an_ACL_denied_node() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717Acl", "Warm", ct); + var raw = rig.EnsureRawVariable("secret", "Double", writable: false); + rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + rig.Enforce(true, [Key()]); // anonymous holds no grant + + using var session = await rig.ConnectAsync(anonymous: true, ct); + (await ReadAsync(session, Raw(session, raw), ct)).StatusCode.ShouldBe(StatusCodes.BadUserAccessDenied); + } + + private sealed class RecordingWriteGateway : IOpcUaNodeWriteGateway + { + public int Count; + + public Task WriteAsync(string nodeId, object? value, AddressSpaceRealm realm, CancellationToken ct) + { + Interlocked.Increment(ref Count); + return Task.FromResult(new NodeWriteOutcome(true, null)); + } + } + + [Theory] + [InlineData("Warm", true)] + [InlineData("Warm", false)] + [InlineData("Unset", true)] + public async Task A_write_on_a_Warm_standby_is_refused_BadInvalidState_and_never_reaches_the_gateway(string mode, bool standby) + { + var ct = Deadline(); + await using var rig = await StartModeAsync($"Warm717Write{mode}{standby}", mode, ct); + var gateway = new RecordingWriteGateway(); + rig.Nm.NodeWriteGateway = gateway; + rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); + var raw = rig.EnsureRawVariable("w", "Double", writable: true); + rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + if (mode == "Warm" && !standby) Promote(rig.Server.CreateStandbyProjection()!); + // Unset with "standby": a node that is not the Primary takes today's optimistic path (no gate wired here). + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var result = await ((SessionClient)session).WriteAsync(H(), new[] + { + new WriteValue { NodeId = Raw(session, raw), AttributeId = Attributes.Value, Value = new DataValue(new Variant(2.0)) }, + }, ct); + + if (mode == "Warm" && standby) + { + result.Results[0].ShouldBe(StatusCodes.BadInvalidState); + await Task.Delay(200, ct); + gateway.Count.ShouldBe(0, "a refused write never reaches the driver"); + } + else + { + result.Results[0].ShouldBe(StatusCodes.Good); + await RecordedEvents.WaitUntilAsync(() => Volatile.Read(ref gateway.Count) == 1); + } + } + + // ------------------------------------------------------------------ T9 Part 9 method refusal + + [Theory] + [InlineData("Warm", true, "Acknowledge")] + [InlineData("Warm", true, "OneShotShelve")] + [InlineData("Warm", false, "Acknowledge")] + [InlineData("Unset", true, "Acknowledge")] + public async Task A_Part9_method_on_a_Warm_standby_is_refused_BadInvalidState_before_any_state_change( + string mode, bool standby, string method) + { + var ct = Deadline(); + await using var rig = await StartModeAsync($"Warm717Method{mode}{standby}{method}", mode, ct); + rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); + rig.MaterialiseScripted(); + var routed = 0; + rig.Nm.AlarmCommandRouter = _ => Interlocked.Increment(ref routed); + var projection = rig.Server.CreateStandbyProjection(); + if (projection is not null) Promote(projection); // project the condition while Active so it has an EventId + rig.Project(native: false, ConditionMethodEventTests.PreState(method, native: false) with { Message = "base" }); + if (projection is not null && standby) Demote(projection); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var eventId = await ReadEventIdAsync(session, ct); + var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), method, eventId, ct); + if (mode == "Warm" && standby) + { + (await Should.ThrowAsync(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState); + routed.ShouldBe(0, "nothing is routed to the engine"); + var acked = await TranslateAsync(session, ConditionNodeId(session, native: false), "AckedState/Id", ct); + (await ReadAsync(session, acked, ct)).ClrValue().ShouldBe(false, "no condition state changed"); + } + else + { + await call; + routed.ShouldBe(1); + } + } + + private static async Task ReadEventIdAsync(ISession session, CancellationToken ct) + { + var eventId = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.EventId, ct); + return ((ByteString)(await ReadAsync(session, eventId, ct)).WrappedValue.AsBoxedObject()!).ToArray()!; + } + + private static async Task TranslateAsync(ISession session, NodeId start, string path, CancellationToken ct) + { + var elements = path.Split('/').Select(name => new RelativePathElement + { + ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, TargetName = new QualifiedName(name), + }).ToArray(); + var r = await ((SessionClient)session).TranslateBrowsePathsToNodeIdsAsync(H(), new[] + { + new BrowsePath { StartingNode = start, RelativePath = new RelativePath { Elements = elements } }, + }, ct); + r.Results[0].StatusCode.ShouldBe(StatusCodes.Good, path); + return ExpandedNodeId.ToNodeId(r.Results[0].Targets[0].TargetId, session.NamespaceUris); + } + + // ------------------------------------------------------------------ T9 alarm suppression + refresh + + [Fact] + public async Task Standby_suppresses_new_alarm_events_on_every_item_kind_and_promotion_refreshes_them_all() + { + var ct = Deadline(120); + await using var rig = await StartModeAsync("Warm717Alarms", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); + rig.MaterialiseNative(); + rig.MaterialiseScripted(); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server); + var folderItem = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA)); + var deviceItem = RecordedEvents.AddEventItem(subscription, + new NodeId(RawDeviceFolder, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri))); + await subscription.ApplyChangesAsync(ct); + + // While Standby (boot state): a new transition reaches no item, but the condition node is current. + rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "while-standby")); + // A condition raised and cleared (Retain=false) while Standby is not replayed by the promotion refresh. + rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash")); + rig.Project(native: false, new AlarmConditionSnapshot(Active: false, Acknowledged: true, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash-cleared")); + await Task.Delay(1000, ct); + foreach (var item in new[] { serverItem, folderItem, deviceItem }) + { + events.Has(item, "while-standby").ShouldBeFalse($"no alarm event on {item.StartNodeId} while Standby"); + events.Has(item, "flash").ShouldBeFalse(); + } + + // D5: a client ConditionRefresh on the Standby returns Start, the retained condition, End. + await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType)); + events.Has(serverItem, "while-standby").ShouldBeTrue("the Standby's refresh carries the retained condition"); + events.Has(serverItem, "flash-cleared").ShouldBeFalse("a non-retained condition is not replayed"); + foreach (var item in new[] { folderItem, deviceItem }) + await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType)); + var countsBefore = new[] { serverItem, folderItem, deviceItem }.ToDictionary(i => i, i => events.Of(i).Count); + + Promote(projection); + foreach (var item in new[] { serverItem, folderItem, deviceItem }) + { + await RecordedEvents.WaitUntilAsync(() => events.Of(item).Skip(countsBefore[item]) + .Any(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType)); + var stream = events.Of(item).Skip(countsBefore[item]).ToList(); + var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType); + var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType); + start.ShouldBeGreaterThanOrEqualTo(0, $"RefreshStart on {item.StartNodeId}"); + var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList(); + replayed.ShouldContain("while-standby", $"promotion refresh replays the retained condition on {item.StartNodeId}"); + replayed.ShouldNotContain("flash-cleared"); + } + + // After promotion new events flow, and the Standby-stamped EventId acknowledges. + rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 800, Message: "after")); + await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "after") && events.Has(folderItem, "after") && events.Has(deviceItem, "after")); + await subscription.DeleteAsync(true, ct); + } + + [Fact] + public async Task A_Standby_stamped_EventId_is_accepted_by_Acknowledge_after_promotion() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717AckAfter", "Warm", ct); + rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); + rig.MaterialiseScripted(); + var routed = new List(); + rig.Nm.AlarmCommandRouter = cmd => { lock (routed) routed.Add(cmd); }; + rig.Project(native: false, ConditionMethodEventTests.PreState("Acknowledge", native: false) with { Message = "stamped" }); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var stamped = await ReadEventIdAsync(session, ct); // stamped while Standby, never delivered + Promote(rig.Server.CreateStandbyProjection()!); + await ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), "Acknowledge", stamped, ct); + lock (routed) routed.ShouldHaveSingleItem().Operation.ShouldBe("Acknowledge"); + } + + // ------------------------------------------------------------------ T8 live items on transition + + [Fact] + public async Task Demotion_queues_one_BadOutOfService_then_silence_and_promotion_queues_the_current_value_through_a_deadband() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717Live", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + var raw = rig.EnsureRawVariable("live", "Double", writable: false); + rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + Promote(projection); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + var client = (SessionClient)session; + var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId; + await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct); + await AddItemAsync(client, sub, Raw(session, raw), 2, 10, ct, new DataChangeFilter + { + Trigger = DataChangeTrigger.StatusValue, DeadbandType = (uint)DeadbandType.Absolute, DeadbandValue = 1000, + }); + (await CollectAsync(client, ct, TimeSpan.FromSeconds(1))).Where(x => x.Handle is 1 or 2).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good); + + Demote(projection); + rig.Sink.WriteValue(raw, 2.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + rig.Sink.WriteValue(raw, 3.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + var demoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2)); + demoted.Count(x => x.Handle == 1).ShouldBe(1, "exactly one notification per item, then silence"); + demoted.Count(x => x.Handle == 2).ShouldBe(1); + demoted.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService); + + rig.Sink.WriteValue(raw, 4.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + Promote(projection); + var promoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2)); + promoted.ShouldContain(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0)); + promoted.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0), + "the promotion value is queued ignoring the deadband"); + } + + // ------------------------------------------------------------------ T7 first-delivery decision record (D4) + + /// Timing 1 (T0): Good samples queued in the item while Active, a client-handle change, then demotion. + [Fact] + public async Task Timing1_queued_good_samples_then_demotion_first_delivery_is_projected_and_republish_matches() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717T1", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + var raw = rig.EnsureRawVariable("q", "Double", writable: false); + rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + rig.MaterialiseScripted(); + Promote(projection); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + var client = (SessionClient)session; + var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId; + var tagItem = await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct); + await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct); + var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct); + await AddItemAsync(client, sub, severity, 3, 1, ct); + (await PublishDataAsync(client, ct)).NotificationMessage.NotificationData.Count.ShouldBeGreaterThan(0); + + await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct); + rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + // Non-ordinary changes queued in the same window: i=2267 and a condition child (Severity). + new SdkServiceLevelPublisher(rig.Server.CurrentInstance!, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance).Publish(90); + rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 650, Message: "child")); + await client.ModifyMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[] + { + new MonitoredItemModifyRequest { MonitoredItemId = tagItem, + RequestedParameters = new MonitoringParameters { ClientHandle = 99, SamplingInterval = 0, QueueSize = 10, DiscardOldest = true } }, + }, ct); + DemoteDeliveryOnly(projection); + await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct); + var msg = await PublishDataAsync(client, ct); + var changes = DataChanges(msg.NotificationMessage); + changes.Where(x => x.Handle is 1 or 99).ShouldNotBeEmpty(); + changes.Where(x => x.Handle is 1 or 99).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService && x.Value.WrappedValue.IsNull, + "Good samples queued before demotion are first-delivered projected (D4)"); + changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not an ordinary tag"); + changes.ShouldContain(x => x.Handle == 3 && x.Value.StatusCode == StatusCodes.Good, "a condition child is not an ordinary tag"); + + var again = await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct); + DataChanges(again.NotificationMessage).Where(x => x.Handle is 1 or 99) + .ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService, "Republish returns the recorded projected choice"); + } + + /// Timing 2 (T0): messages prebuilt while Active (one notification per message), then demotion. + [Fact] + public async Task Timing2_prebuilt_messages_then_demotion_next_publish_is_projected_and_unsent_republish_is_unavailable() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717T2", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + var raw = rig.EnsureRawVariable("p", "Double", writable: false); + rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + Promote(projection); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + var client = (SessionClient)session; + var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 1, true, 0, ct)).SubscriptionId; + await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct); + DataChanges((await PublishDataAsync(client, ct)).NotificationMessage).Count.ShouldBe(1); + + await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct); + rig.Sink.WriteValue(raw, 20.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + rig.Sink.WriteValue(raw, 21.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + rig.Sink.WriteValue(raw, 22.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct); + var a = await PublishDataAsync(client, ct); // builds several messages while Active; the first is delivered Good + DataChanges(a.NotificationMessage).Single().Value.StatusCode.ShouldBe(StatusCodes.Good); + + DemoteDeliveryOnly(projection); + var next = a.NotificationMessage.SequenceNumber + 1; + (await Should.ThrowAsync(client.RepublishAsync(H(), sub, next, ct).AsTask())).StatusCode + .ShouldBe(StatusCodes.BadMessageNotAvailable, "2.0.1: a prebuilt, never-sent message cannot be first-delivered by Republish"); + var b = await PublishDataAsync(client, ct); + b.NotificationMessage.SequenceNumber.ShouldBe(next); + DataChanges(b.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService, + "a message built while Active but first delivered after demotion is projected"); + var bRe = await client.RepublishAsync(H(), sub, next, ct); + DataChanges(bRe.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService); + } + + /// Timing 3 (T0): Good delivered while Active, then demotion — Republish returns the original Good; after an + /// acknowledge it is gone. + [Fact] + public async Task Timing3_good_delivered_while_active_republishes_good_after_demotion_and_ack_drops_it() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717T3", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + var raw = rig.EnsureRawVariable("d", "Double", writable: false); + rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + Promote(projection); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + var client = (SessionClient)session; + var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId; + await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct); + var first = await PublishDataAsync(client, ct); + DataChanges(first.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good); + + Demote(projection); + var again = await client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct); + DataChanges(again.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good, + "Republish of a message first delivered while Active returns the original (D4 boundary)"); + + await client.PublishAsync(H(), new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = first.NotificationMessage.SequenceNumber } }, ct); + (await Should.ThrowAsync(client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct).AsTask())) + .StatusCode.ShouldBe(StatusCodes.BadMessageNotAvailable); + } + + /// The D4 boundary alone (spec §4.7 demotion step 2), without the projection that follows it, so the + /// decision record is what the test observes. + private static void DemoteDeliveryOnly(IStandbyProjection p) => p.SetStandbyDelivery(true); + + // ------------------------------------------------------------------ raw client helpers + + private static async Task AddItemAsync(SessionClient client, uint sub, NodeId node, uint handle, uint queue, CancellationToken ct, + MonitoringFilter? filter = null) + { + var r = await client.CreateMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[] + { + new MonitoredItemCreateRequest + { + ItemToMonitor = new ReadValueId { NodeId = node, AttributeId = Attributes.Value }, + MonitoringMode = MonitoringMode.Reporting, + RequestedParameters = new MonitoringParameters + { + ClientHandle = handle, SamplingInterval = 0, QueueSize = queue, DiscardOldest = true, + Filter = filter is null ? ExtensionObject.Null : new ExtensionObject(filter), + }, + }, + }, ct); + StatusCode.IsGood(r.Results[0].StatusCode).ShouldBeTrue($"create {node}: {r.Results[0].StatusCode}"); + return r.Results[0].MonitoredItemId; + } + + internal static List<(uint Handle, DataValue Value)> DataChanges(NotificationMessage m) + { + var list = new List<(uint, DataValue)>(); + for (var i = 0; i < m.NotificationData.Count; i++) + { + if (m.NotificationData[i].TryGetValue(out var d)) + for (var j = 0; j < d.MonitoredItems.Count; j++) list.Add((d.MonitoredItems[j].ClientHandle, d.MonitoredItems[j].Value)); + } + + return list; + } + + private static async Task PublishDataAsync(SessionClient client, CancellationToken ct) + { + for (var i = 0; i < 40; i++) + { + var r = await client.PublishAsync(H(), default, ct); + if (r.NotificationMessage.NotificationData.Count > 0) return r; + } + + throw new InvalidOperationException("no data message"); + } + + private static async Task> CollectAsync(SessionClient client, CancellationToken ct, TimeSpan window) + { + var collected = new List<(uint, DataValue)>(); + var until = DateTime.UtcNow + window; + while (DateTime.UtcNow < until) + { + var r = await client.PublishAsync(H(), default, ct); + collected.AddRange(DataChanges(r.NotificationMessage)); + } + + return collected; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/WarmStandbyUnitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/WarmStandbyUnitTests.cs new file mode 100644 index 000000000..28eed6635 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/WarmStandbyUnitTests.cs @@ -0,0 +1,250 @@ +using Opc.Ua; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests; + +/// +/// Gitea #717 spec §4.6 (the T0-adopted mechanism) at unit level: the ordinary-tag marker and its validation, the +/// Standby rewrite, the (SubscriptionId, SequenceNumber) decision record and its bounds, and the redundancy option +/// rules of spec §3. +/// +public sealed class WarmStandbyUnitTests +{ + private static DataValue Good(double v, int secondsAgo = 0) => + new(new Variant(v), StatusCodes.Good, (DateTimeUtc)DateTime.UtcNow.AddSeconds(-secondsAgo), DateTimeUtc.Now); + + private static MonitoredItemNotification Marked(uint handle, DataValue value) + { + var n = new MonitoredItemNotification { ClientHandle = handle, Value = value }; + n.Message = new OrdinaryTagMarker(handle, value); + return n; + } + + private static NotificationMessage Message(uint seq, params MonitoredItemNotification[] items) => new() + { + SequenceNumber = seq, + PublishTime = DateTimeUtc.Now, + NotificationData = new List + { + new(new DataChangeNotification { MonitoredItems = items.ToList() }), + }, + }; + + private static List Items(NotificationMessage m) + { + m.NotificationData[0].TryGetValue(out var d).ShouldBeTrue(); + return Enumerable.Range(0, d!.MonitoredItems.Count).Select(i => d.MonitoredItems[i]).ToList(); + } + + // ------------------------------------------------------------------ marker + rewrite + + [Fact] + public void A_marked_ordinary_tag_notification_is_rewritten_and_the_input_is_not_mutated() + { + var value = Good(5); + var tag = Marked(1, value); + var other = new MonitoredItemNotification { ClientHandle = 2, Value = Good(6) }; + var input = Message(7, tag, other); + + var projected = StandbyNotificationRewriter.Project(input, out var rewritten); + + rewritten.ShouldBe(1); + ReferenceEquals(projected, input).ShouldBeFalse(); + var items = Items(projected); + items[0].ClientHandle.ShouldBe(1u); + items[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService); + items[0].Value.WrappedValue.IsNull.ShouldBeTrue(); + items[0].Value.ServerTimestamp.ShouldBe(value.ServerTimestamp); + ReferenceEquals(items[1], other).ShouldBeTrue("a non-ordinary notification is passed through untouched"); + projected.SequenceNumber.ShouldBe(7u); + + // The SDK-owned input (the per-delivery clone) is untouched. + Items(input)[0].Value.StatusCode.ShouldBe(StatusCodes.Good); + ReferenceEquals(Items(input)[0], tag).ShouldBeTrue(); + } + + [Fact] + public void A_stale_marker_from_a_recycled_payload_is_ignored() + { + // 2.0.1 pools MonitoredItemNotification payloads and Initialize() resets only ClientHandle and Value, so a + // recycled object can carry a marker made for another notification. Handle or value mismatch ⇒ not ordinary. + var otherHandle = new MonitoredItemNotification { ClientHandle = 3, Value = Good(5) }; + otherHandle.Message = new OrdinaryTagMarker(1, otherHandle.Value); + var otherValue = new MonitoredItemNotification { ClientHandle = 1, Value = Good(6) }; + otherValue.Message = new OrdinaryTagMarker(1, Good(5, secondsAgo: 10)); + + OrdinaryTagMarker.IsOrdinaryTag(otherHandle).ShouldBeFalse(); + OrdinaryTagMarker.IsOrdinaryTag(otherValue).ShouldBeFalse(); + var input = Message(1, otherHandle, otherValue); + ReferenceEquals(StandbyNotificationRewriter.Project(input, out var rewritten), input).ShouldBeTrue(); + rewritten.ShouldBe(0); + } + + [Fact] + public void Bad_and_unmarked_notifications_are_never_rewritten() + { + var bad = Marked(1, new DataValue(Variant.Null, StatusCodes.BadNoCommunication, DateTimeUtc.Now, DateTimeUtc.Now)); + var unmarked = new MonitoredItemNotification { ClientHandle = 2, Value = Good(1) }; + var input = Message(1, bad, unmarked); + ReferenceEquals(StandbyNotificationRewriter.Project(input, out _), input).ShouldBeTrue(); + } + + [Fact] + public void Uncertain_ordinary_tag_values_are_projected_too() + { + var uncertain = Marked(1, new DataValue(new Variant(1.0), StatusCodes.UncertainLastUsableValue, DateTimeUtc.Now, DateTimeUtc.Now)); + Items(StandbyNotificationRewriter.Project(Message(1, uncertain), out _))[0].Value.StatusCode.ShouldBe(StatusCodes.BadOutOfService); + } + + // ------------------------------------------------------------------ SDK-field contract (regression pin) + + /// + /// The mechanism stands on two SDK 2.0.1 behaviours of : the SDK's + /// per-delivery CoreUtils.Clone COPIES it, and the binary encoder NEVER writes it. An SDK upgrade that + /// changes either breaks D4 silently (no marker ⇒ no rewrite) or leaks a server object onto the wire — this + /// fails loudly instead. + /// + [Fact] + public void Sdk_contract_the_marker_survives_the_delivery_clone_and_is_never_encoded() + { + var value = Good(5); + var marked = Message(9, Marked(1, value)); + + var clone = CoreUtils.Clone(marked); + ReferenceEquals(clone, marked).ShouldBeFalse(); + var cloned = Items(clone!)[0]; + ReferenceEquals(cloned, Items(marked)[0]).ShouldBeFalse("the clone is deep"); + cloned.Message.ShouldBeOfType("the SDK clone must copy MonitoredItemNotification.Message"); + OrdinaryTagMarker.IsOrdinaryTag(cloned).ShouldBeTrue("the copied marker still validates against the clone"); + + var context = ServiceMessageContext.Create(DefaultTelemetry.Create(_ => { })); + var unmarked = Message(9, new MonitoredItemNotification { ClientHandle = 1, Value = value }); + unmarked.PublishTime = marked.PublishTime; + var withMarker = BinaryEncoder.EncodeMessage(marked, context); + var without = BinaryEncoder.EncodeMessage(unmarked, context); + withMarker.ShouldBe(without, "the helper field must never reach the wire"); + var decoded = BinaryDecoder.DecodeMessage(withMarker, context); + Items(decoded!)[0].Message.ShouldBeNull(); + } + + // ------------------------------------------------------------------ decision record + + [Fact] + public void The_first_delivery_records_the_decision_and_every_later_delivery_reuses_it() + { + var record = new FirstDeliveryRecord(standby: false); + record.Decide(1, 10).ShouldBeFalse(); + record.SetStandby(true); + record.Decide(1, 10).ShouldBeFalse("a Republish of a message first delivered Active stays Active (D4)"); + record.Decide(1, 11).ShouldBeTrue("a first delivery after the flip is Standby"); + record.SetStandby(false); + record.Decide(1, 11).ShouldBeTrue("a Republish of a message first delivered Standby stays projected"); + record.Decide(2, 11).ShouldBeFalse("keyed by subscription too"); + } + + [Fact] + public void The_record_is_pruned_by_available_sequence_numbers_and_bounded() + { + var record = new FirstDeliveryRecord(standby: true); + for (uint s = 1; s <= 5; s++) record.Decide(1, s); + record.Prune(1, [3, 4], justDelivered: 5); + record.Count.ShouldBe(3); + + for (uint s = 100; s < 100 + FirstDeliveryRecord.MaxPerSubscription + 50; s++) record.Decide(1, s); + record.Count.ShouldBe(FirstDeliveryRecord.MaxPerSubscription); + + record.Decide(2, 1); + record.SweepDead(new HashSet { 2 }); + record.SubscriptionCount.ShouldBe(1); + } + + [Fact] + public void Dead_subscriptions_are_evicted_least_recently_used_beyond_the_bound() + { + var record = new FirstDeliveryRecord(standby: false); + for (uint sub = 1; sub <= FirstDeliveryRecord.MaxSubscriptions; sub++) record.Decide(sub, 1); + record.Decide(1, 2); // touch subscription 1 so 2 is the least recently used + record.Decide(FirstDeliveryRecord.MaxSubscriptions + 1, 1); + record.SubscriptionCount.ShouldBe(FirstDeliveryRecord.MaxSubscriptions); + } + + /// + /// A role flip and a first delivery race (spec §4.6 item 2): the flip waits for a decision in progress, so the + /// decision is ordered strictly before or after it — never torn. Driven by a deterministic barrier, not sleeps. + /// + [Fact] + public async Task A_flip_waits_for_a_decision_in_progress_and_orders_strictly_against_it() + { + for (var round = 0; round < 50; round++) + { + var record = new FirstDeliveryRecord(standby: false); + using var inside = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + record.DecisionBarrier = () => { inside.Set(); release.Wait(TimeSpan.FromSeconds(10)); }; + + var ct = TestContext.Current.CancellationToken; + var decision = Task.Run(() => record.Decide(1, (uint)round + 1), ct); + inside.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue(); + var flip = Task.Run(() => record.SetStandby(true), ct); + (await Task.WhenAny(flip, Task.Delay(100, ct))).ShouldNotBe(flip, "the flip must wait for the decision in progress"); + release.Set(); + (await decision).ShouldBeFalse("the decision read the state before the flip"); + await flip; + record.DecisionBarrier = null; + record.Decide(1, (uint)round + 1).ShouldBeFalse("and recorded it"); + record.Decide(1, (uint)round + 1000).ShouldBeTrue("a later first delivery sees the flip"); + } + } + + // ------------------------------------------------------------------ WarmStandbyState + + [Fact] + public void The_Off_state_never_projects_suppresses_or_refuses() + { + var off = new WarmStandbyState(false); + off.DataProjected = true; + off.AlarmsSuppressed = true; + off.DataProjected.ShouldBeFalse(); + off.AlarmsSuppressed.ShouldBeFalse(); + off.RefusesMutations.ShouldBeFalse(); + off.Delivery.ShouldBeNull(); + + var warm = new WarmStandbyState(true); + warm.DataProjected.ShouldBeTrue("a Warm node boots Standby"); + warm.AlarmsSuppressed.ShouldBeTrue(); + warm.RefusesMutations.ShouldBeTrue(); + warm.Delivery!.Standby.ShouldBeTrue(); + } + + // ------------------------------------------------------------------ OpcUa:Redundancy rules (spec §3) + + [Theory] + [InlineData("Unset", new string[0], true)] + [InlineData("", new string[0], true)] + [InlineData("none", new string[0], true)] + [InlineData("None", new[] { "urn:peer" }, false)] + [InlineData("Warm", new[] { "urn:peer" }, true)] + [InlineData("WARM", new[] { "urn:peer", "urn:peer2" }, true)] + [InlineData("Warm", new string[0], false)] + [InlineData("Warm", new[] { "urn:peer", "urn:peer" }, false)] + [InlineData("Warm", new[] { "urn:self" }, false)] + [InlineData("Hot", new[] { "urn:peer" }, false)] + [InlineData("2", new[] { "urn:peer" }, false)] + [InlineData("Transparent", new[] { "urn:peer" }, false)] + public void Redundancy_option_rules(string mode, string[] peers, bool valid) + { + var options = new OpcUaRedundancyOptions { Mode = mode }; + options.Check("urn:self", peers).Count.ShouldBe(valid ? 0 : 1, string.Join("; ", options.Check("urn:self", peers))); + } + + [Fact] + public void Redundancy_defaults_are_Unset_with_promote_on() + { + var options = new OpcUaApplicationHostOptions(); + options.Redundancy.ParsedMode.ShouldBe(OpcUaRedundancyMode.Unset); + options.Redundancy.PromoteWhilePrimaryUnreachable.ShouldBeTrue(); + options.Redundancy.ToSettings().ShouldBe(WarmRedundancySettings.Unset); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorWarmTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorWarmTests.cs new file mode 100644 index 000000000..639c7270a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorWarmTests.cs @@ -0,0 +1,220 @@ +using System.Collections.Concurrent; +using System.Text; +using Akka.Actor; +using Akka.TestKit; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa; + +/// +/// Gitea #717 spec §4.1–§4.3 and §4.7 in : the Warm verdict from membership + the +/// accepted revisioned snapshot, the bands, and the transition step order — with a recording projection and +/// ServiceLevel publisher writing into ONE ordered log, so the invariant "an Active-band level is never visible over +/// projected data" is checked on the actual sequence. +/// +public sealed class OpcUaPublishActorWarmTests : RuntimeActorTestBase +{ + private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9"); + + private readonly ConcurrentQueue _log = new(); + private long _seq; + + private NodeId Self + { + get + { + var cluster = Akka.Cluster.Cluster.Get(Sys); + AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget); + return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}"); + } + } + + private IActorRef Start(WarmRedundancySettings redundancy, LoggingPublisher? publisher = null) => + Sys.ActorOf(OpcUaPublishActor.PropsForTests( + serviceLevel: publisher ?? new LoggingPublisher(_log), localNode: Self, dbLess: true, + healthTickInterval: TimeSpan.FromMinutes(5), redundancy: redundancy, + standbyProjection: new RecordingProjection(_log))); + + private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) => + new(new[] + { + new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary, + false, primary == Self, DateTime.UtcNow), + new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary, + false, primary == Other, DateTime.UtcNow), + }, + CorrelationId.NewId(), + revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null); + + private string[] Steps => _log.ToArray(); + + private void AwaitLogEndsWith(params string[] tail) => + AwaitAssert(() => Steps.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", Steps)), PresenceBudget, + cancellationToken: TestContext.Current.CancellationToken); + + private static readonly string[] Promotion = ["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL:250"]; + private static readonly string[] Demotion = ["SL:90", "delivery:True", "alarms:True", "proj:True", "queue:True"]; + + [Fact] + public void Promotion_and_demotion_run_in_the_spec_order() + { + var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm)); + actor.Tell(Snapshot(Self)); + AwaitLogEndsWith(Promotion); + Steps.ShouldBe(Promotion, "boot is Standby; the first accepted snapshot promotes and nothing else ran"); + + actor.Tell(Snapshot(Other)); + AwaitLogEndsWith(Demotion); + AssertNoActiveLevelOverProjectedData(); + } + + /// Spec §4.7 invariant over the whole sequence: every published level ≥ 100 comes after the projection was + /// last opened, never while it is closed. + private void AssertNoActiveLevelOverProjectedData() + { + var projected = true; // a Warm node boots projected + foreach (var step in Steps) + { + if (step == "proj:True") projected = true; + else if (step == "proj:False") projected = false; + else if (step.StartsWith("SL:", StringComparison.Ordinal) && byte.Parse(step[3..]) >= 100) + projected.ShouldBeFalse($"an Active-band level was visible over projected data: {string.Join(" ", Steps)}"); + } + } + + [Fact] + public void An_unrevisioned_snapshot_never_makes_a_Warm_node_Active() + { + var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm)); + actor.Tell(Snapshot(Self, revisioned: false)); + AwaitLogEndsWith("SL:90"); + Steps.ShouldBe(new[] { "SL:90" }); + } + + /// Spec §4.2: after a newer snapshot names this node Primary, an OLDER incarnation's late publish naming the + /// peer is dropped — the node stays Active. Then a genuinely newer one demotes it (positive control). + [Fact] + public void An_older_revision_after_a_newer_one_is_dropped() + { + var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm)); + actor.Tell(Snapshot(Self, upNumber: 5)); + AwaitLogEndsWith(Promotion); + + actor.Tell(Snapshot(Other, upNumber: 4)); // a departed older singleton's late heartbeat + actor.Tell(Snapshot(Self, upNumber: 5)); // a fence: processed after the stale one + AwaitAssert(() => Steps.Count(s => s == "SL:250").ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + Steps.ShouldNotContain("delivery:True", "the stale snapshot must not demote"); + + actor.Tell(Snapshot(Other, upNumber: 6)); + AwaitLogEndsWith(Demotion); + } + + /// Spec §4.7: if the Active level cannot be published, the promotion stops before it and retries; the + /// Standby level stays visible meanwhile. + [Fact] + public void A_failed_Active_level_publish_halts_before_it_and_retries() + { + var publisher = new LoggingPublisher(_log) { FailNext = 1, FailLevel = 250 }; + var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher); + actor.Tell(Snapshot(Self)); + AwaitAssert(() => Steps.ShouldContain("SL:250"), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + var log = Steps; + log.Take(6).ShouldBe(["proj:False", "queue:False", "delivery:False", "alarms:False", "refresh", "SL-FAILED:250"]); + log.Last().ShouldBe("SL:250", "retried without a new snapshot"); + } + + /// A demotion continues when the Standby level publish fails: failing safe degrades the data. + [Fact] + public void A_failed_Standby_level_publish_still_degrades_the_data() + { + var publisher = new LoggingPublisher(_log); + var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm), publisher); + actor.Tell(Snapshot(Self)); + AwaitLogEndsWith(Promotion); + publisher.FailNext = 1; + publisher.FailLevel = 90; + actor.Tell(Snapshot(Other)); + AwaitLogEndsWith("SL-FAILED:90", "delivery:True", "alarms:True", "proj:True", "queue:True"); + } + + [Fact] + public void A_burst_of_alternating_verdicts_converges_on_the_last() + { + var actor = Start(new WarmRedundancySettings(OpcUaRedundancyMode.Warm)); + for (var i = 0; i < 20; i++) actor.Tell(Snapshot(i % 2 == 0 ? Self : Other)); + actor.Tell(Snapshot(Self)); + AwaitLogEndsWith(Promotion); + AssertNoActiveLevelOverProjectedData(); + } + + /// The default-off pin at the actor: Unset never touches the projection and keeps today's levels. + [Fact] + public void Unset_never_drives_the_projection_and_keeps_todays_levels() + { + var actor = Start(WarmRedundancySettings.Unset); + actor.Tell(Snapshot(Self)); + AwaitLogEndsWith("SL:250"); + actor.Tell(Snapshot(Other)); + AwaitLogEndsWith("SL:240"); + Steps.ShouldBe(new[] { "SL:250", "SL:240" }); + } + + [Fact] + public void A_deployed_cluster_mode_that_disagrees_with_the_local_mode_is_logged_and_counted() + { + var artifact = Encoding.UTF8.GetBytes( + "{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":\"None\"}],\"Nodes\":[]}"); + DeploymentArtifact.ReadClusterRedundancyMode(artifact, "x:1").ShouldBe("None"); + DeploymentArtifact.ReadClusterRedundancyMode( + Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"c1\",\"RedundancyMode\":1}]}"), "x:1").ShouldBe("Warm"); + DeploymentArtifact.ReadClusterRedundancyMode( + Encoding.UTF8.GetBytes("{\"Clusters\":[{\"ClusterId\":\"a\",\"RedundancyMode\":\"None\"},{\"ClusterId\":\"b\",\"RedundancyMode\":\"Hot\"}]," + + "\"Nodes\":[{\"NodeId\":\"x:1\",\"ClusterId\":\"b\"}]}"), "x:1").ShouldBe("Hot"); + + using var listener = new System.Diagnostics.Metrics.MeterListener(); + long mismatches = 0; + listener.InstrumentPublished = (instrument, l) => + { + if (ReferenceEquals(instrument, ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.RedundancyModeMismatch)) + l.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((_, value, _, _) => Interlocked.Add(ref mismatches, value)); + listener.Start(); + + var warmActor = Sys.ActorOf(OpcUaPublishActor.PropsForTests( + localNode: NodeId.Parse("x:1"), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm))); + EventFilter.Error(contains: "RedundancyMode for this node's cluster is None").ExpectOne(() => + warmActor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), Artifact: artifact)), + TestContext.Current.CancellationToken); + Interlocked.Read(ref mismatches).ShouldBeGreaterThanOrEqualTo(1); + } +} + +/// Writes each published level into the shared log; can be told to fail the next N publishes of a level. +internal sealed class LoggingPublisher(ConcurrentQueue log) : IServiceLevelPublisher +{ + public int FailNext { get; set; } + public byte FailLevel { get; set; } + + public void Publish(byte serviceLevel) + { + if (FailNext > 0 && serviceLevel == FailLevel) + { + FailNext--; + log.Enqueue($"SL-FAILED:{serviceLevel}"); + throw new InvalidOperationException("injected ServiceLevel publish failure"); + } + + log.Enqueue($"SL:{serviceLevel}"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Redundancy/WarmRoleAuthorityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Redundancy/WarmRoleAuthorityTests.cs new file mode 100644 index 000000000..3615d4fa8 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Redundancy/WarmRoleAuthorityTests.cs @@ -0,0 +1,91 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy; + +/// Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7). +public sealed class WarmRoleAuthorityTests +{ + private static readonly NodeId Self = NodeId.Parse("a:1"); + private static readonly NodeId Peer = NodeId.Parse("b:1"); + + private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true, + NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) => + new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote); + + [Fact] + public void Active_iff_membership_and_the_accepted_snapshot_both_name_self() + { + WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active); + WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf) + { + var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self)); + verdict.Role.ShouldBe(WarmRole.Standby); + verdict.Disagreement.ShouldBeTrue(); + } + + [Fact] + public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership() + { + var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null)); + verdict.Role.ShouldBe(WarmRole.Standby); + verdict.Reason.ShouldBe("boot"); + } + + [Fact] + public void Not_up_or_endpoint_failed_is_Standby() + { + WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby); + WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby); + } + + /// Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes. + [Fact] + public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option() + { + var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true); + WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active); + WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby); + WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby, + "a reachable Primary is never overridden"); + WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role + .ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby"); + } + + [Fact] + public void A_late_bind_replays_the_retained_state_but_not_the_actions() + { + var deferred = new DeferredStandbyProjection(); + deferred.SetDataProjection(false); + deferred.SetStandbyDelivery(false); + deferred.SetAlarmSuppression(false); + deferred.QueueCurrentToLiveItems(false).ShouldBe(0); + deferred.RefreshAllEventItems().ShouldBe(0); + + var inner = new RecordingProjection(); + deferred.SetInner(inner); + inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" }); + deferred.QueueCurrentToLiveItems(true); + inner.Log.Last().ShouldBe("queue:True"); + } +} + +/// Records every step into a shared, ordered log. +internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue? log = null) : IStandbyProjection +{ + public System.Collections.Concurrent.ConcurrentQueue Log { get; } = log ?? new(); + public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}"); + public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}"); + public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}"); + public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; } + public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; } +}