feat(717): Warm non-transparent redundancy, default-off (T1-T10)

OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start).
- T1 options + validator + effective-URI check + boot banner; Warm needs the driver role.
- T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only
  ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None
  and the SDK's orphan i=11314 deleted; Unset untouched.
- T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older
  revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active.
- T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable.
- T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec
  4.7 order (Active level last and retried; Standby level first, data degraded regardless).
- T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState.
- T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by
  (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's
  cached copy is never mutated.
- T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters.
- T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion
  refresh via ISubscription.ConditionRefresh2Async on every live event item.
- T10 deploy-time RedundancyMode cross-check + six instruments.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:26:36 -05:00
parent dcff5ae994
commit 7829b937d1
27 changed files with 3056 additions and 29 deletions
@@ -37,6 +37,11 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
private readonly Action<object>? _broadcastOverride;
private readonly TimeSpan _heartbeatInterval;
private bool _dirty;
/// <summary>Gitea #717 spec §4.2: <c>Environment.TickCount64</c> at <see cref="PreStart"/>, ordering a singleton
/// restarted in the same process.</summary>
private long _incarnationTick;
/// <summary>Gitea #717 spec §4.2: incremented on every publish, heartbeats included.</summary>
private long _sequence;
/// <summary>Gets the timer scheduler for this actor.</summary>
public ITimerScheduler Timers { get; set; } = null!;
@@ -78,6 +83,7 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
/// <inheritdoc />
protected override void PreStart()
{
_incarnationTick = NextIncarnationTick();
_cluster.Subscribe(
Self,
ClusterEvent.InitialStateAsEvents,
@@ -107,12 +113,63 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
_dirty = false;
var snapshot = BuildSnapshot();
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId());
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId(), NextRevision());
if (_broadcastOverride is not null) _broadcastOverride(msg);
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(Topic, msg));
_log.Debug("Published RedundancyStateChanged with {Count} nodes", snapshot.Count);
}
/// <summary>
/// The ordered revision of the next publish (Gitea #717 spec §4.2): the up-number of the member hosting this
/// singleton, this incarnation's start tick, and a per-publish sequence. Subscribers drop anything not newer,
/// so a late heartbeat from an old incarnation can no longer overwrite the new incarnation's snapshot.
/// </summary>
private SnapshotRevision NextRevision()
{
long upNumber;
try
{
upNumber = ReadUpNumber(_cluster.SelfMember)
?? throw new InvalidOperationException("Akka.Cluster Member.UpNumber is not readable on this Akka version");
}
catch (Exception ex)
{
// Not expected on a running singleton; an unordered 0 can never outrank a real publisher.
_log.Debug(ex, "RedundancyState: SelfMember up-number unavailable; publishing revision up-number 0");
upNumber = 0;
}
return new SnapshotRevision(upNumber, _incarnationTick, ++_sequence);
}
private static long _lastIncarnationTick;
/// <summary><c>Environment.TickCount64</c>, made strictly increasing within the process so a singleton restarted
/// inside one tick still outranks its predecessor (whose sequence may be ahead of the new one's).</summary>
/// <returns>The incarnation tick.</returns>
public static long NextIncarnationTick()
{
while (true)
{
var last = Interlocked.Read(ref _lastIncarnationTick);
var next = Math.Max(Environment.TickCount64, last + 1);
if (Interlocked.CompareExchange(ref _lastIncarnationTick, next, last) == last) return next;
}
}
private static readonly System.Reflection.PropertyInfo? UpNumberProperty = typeof(Member).GetProperty(
"UpNumber", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic);
/// <summary>
/// A member's up-number — the order in which members became Up, the basis of <c>Member.IsOlderThan</c> and of
/// singleton placement. Akka.NET 1.5 keeps the property <c>internal</c>, so it is read by reflection; the
/// <c>SnapshotRevisionTests</c> pin that it resolves, so an Akka upgrade that renames it fails loudly.
/// </summary>
/// <param name="member">The member.</param>
/// <returns>The up-number, or null if the property is unavailable.</returns>
public static int? ReadUpNumber(Member member) =>
UpNumberProperty?.GetValue(member) is int value ? value : null;
/// <summary>The cluster role that marks a node as carrying the driver runtime.</summary>
public const string DriverRole = RoleParser.Driver;
@@ -60,5 +60,10 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase<
|| lockout.MaxFailedAttemptsPerClient >= lockout.MaxFailedAttempts,
"OpcUa:CredentialLockout:MaxFailedAttemptsPerClient must be 0 (= 2 x MaxFailedAttempts) or at least "
+ "OpcUa:CredentialLockout:MaxFailedAttempts");
// Gitea #717 spec §3: Hot/unknown modes fail start; Warm needs distinct peers that do not include this server's
// own ApplicationUri; None needs no peers. The effective URI is re-checked after an ApplicationConfigPath load.
foreach (var failure in (o.Redundancy ?? new OpcUaRedundancyOptions()).Check(o.ApplicationUri, o.PeerApplicationUris))
builder.RequireThat(false, failure);
}
}
@@ -0,0 +1,27 @@
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// The one Gitea #717 startup rule the options validator cannot see: <c>OpcUa:Redundancy:Mode=Warm</c> needs the
/// <c>driver</c> role (the validator is registered only on driver nodes, so on any other node it would never run).
/// </summary>
public static class OpcUaRedundancyStartupGuard
{
/// <summary>Fails host start when a node without the driver role is configured Warm.</summary>
/// <param name="configuredMode">The raw <c>OpcUa:Redundancy:Mode</c> value.</param>
/// <param name="hasDriver">Whether this node carries the driver role.</param>
/// <exception cref="InvalidOperationException">Warm without the driver role.</exception>
public static void RequireDriverRoleForWarm(string? configuredMode, bool hasDriver)
{
if (hasDriver) return;
var options = new OpcUaRedundancyOptions { Mode = configuredMode ?? string.Empty };
if (options.TryParseMode(out var mode) && mode == OpcUaRedundancyMode.Warm)
{
throw new InvalidOperationException(
"OpcUa:Redundancy:Mode is Warm on a node without the driver role; Warm redundancy needs the driver role "
+ "(the OPC UA endpoint is served by driver nodes only). Remove the setting or add the driver role.");
}
}
}
@@ -36,6 +36,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
private readonly OpcUaApplicationHostOptions _options;
private readonly DeferredAddressSpaceSink _deferredSink;
private readonly DeferredServiceLevelPublisher _deferredServiceLevel;
private readonly DeferredStandbyProjection? _deferredStandby;
private readonly IOpcUaUserAuthenticator _userAuthenticator;
private readonly IHistorianDataSource _historianDataSource;
private readonly IDataPlaneAuthoritySource _dataPlaneAuthority;
@@ -127,6 +128,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
/// <param name="writeBounds">The node's configured write bounds (<c>DriverWrites:Timeout</c>); the inbound
/// write gateway's Ask is derived from them (lmxopcua#777). Optional: null ⇒
/// <see cref="DriverWriteBounds.Default"/>.</param>
/// <param name="deferredStandby">The late-bound Warm standby projection (Gitea #717) <c>OpcUaPublishActor</c> drives;
/// bound here once the SDK has started. Optional: null (and a non-Warm server) binds nothing.</param>
public OtOpcUaServerHostedService(
IOptions<OpcUaApplicationHostOptions> options,
DeferredAddressSpaceSink deferredSink,
@@ -147,8 +150,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
OpcUaEndpointState? endpointState = null,
EffectiveRoles? roles = null,
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null)
DriverWriteBounds? writeBounds = null,
DeferredStandbyProjection? deferredStandby = null)
{
_deferredStandby = deferredStandby;
ArgumentNullException.ThrowIfNull(aclOptions);
ArgumentNullException.ThrowIfNull(time);
@@ -402,6 +407,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
serverInternal,
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
}
// Gitea #717: the Warm standby projection. Bound right after the ServiceLevel so a transition the actor
// already decided replays onto the node manager; null (every non-Warm server) leaves the adapter unbound.
if (_server.CreateStandbyProjection() is { } standbyProjection) _deferredStandby?.SetInner(standbyProjection);
}
catch (Exception ex)
{
@@ -568,6 +577,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
// half-disposed NodeManager.
_deferredSink.SetSink(null);
_deferredServiceLevel.SetInner(null);
_deferredStandby?.SetInner(null);
// Restore the Null write gateway so a late client write doesn't Ask a stopping DriverHostActor.
_server?.SetNodeWriteGateway(null);
// Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor.
@@ -206,6 +206,10 @@ ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.OpcUaUserCertificatePaths.Configure(
// lmxopcua#765: refresh/staleness of the cached API-key → Galaxy ack identity set (driver nodes consume it).
builder.Services.AddGalaxyAckIdentityCache(builder.Configuration);
// Gitea #717 spec §3: Warm redundancy is a property of the OPC UA endpoint, which only a driver node serves. A Warm
// mode on a node without the driver role is a configuration error, not something to ignore silently.
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(builder.Configuration["OpcUa:Redundancy:Mode"], hasDriver);
if (hasDriver)
{
builder.Services.AddOtOpcUaRuntime();
@@ -501,6 +505,11 @@ if (hasDriver)
builder.Services.AddValidatedOptions<OpcUaApplicationHostOptions, OpcUaApplicationHostOptionsValidator>(
builder.Configuration, "OpcUa");
// Gitea #717: the runtime half of OpcUa:Redundancy for the redundancy actors, and the late-bound standby
// projection the OPC UA hosted service binds once the SDK has started (same pattern as the ServiceLevel publisher).
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<OpcUaApplicationHostOptions>>().Value.Redundancy.ToSettings());
builder.Services.AddSingleton<DeferredStandbyProjection>();
// Lazy ActorSystem accessor so OtOpcUaServerHostedService can resolve the DistributedPubSub
// mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
// DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
@@ -6,8 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// Preserves the application's established equipment-write conversions across SDK 2.0's
/// new pre-handler type validation. The owning callback remains the authorization and shape gate.
/// </summary>
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock) : BaseDataVariableState(parent)
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock, WarmStandbyState? standby = null)
: BaseDataVariableState(parent)
{
private readonly WarmStandbyState _standby = standby ?? WarmStandbyState.Off;
// Every field below is accessed under ownerLock. Tokens identify client attempts; the mutation
// revision detects a driver publication that re-enters the callback before its optimistic commit.
private long _nextWriteToken;
@@ -82,6 +85,16 @@ internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock
QualifiedName dataEncoding, ref Variant value, ref DateTimeUtc sourceTimestamp)
{
// Subscription samplers read the node directly, bypassing the manager's service lock.
lock (ownerLock) return base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
ServiceResult result;
lock (ownerLock) result = base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
// Gitea #717 spec §4.4 (T0b): on a Warm Standby every value path — Read (IndexRange and DataEncoding included),
// the initial sample, the change fan-out, ResendData, AggregateFilter input — answers BadOutOfService/null. The
// stored Value is untouched and keeps updating from acquisition. The base read runs first, so an IndexRange or
// encoding error is still reported as such. Off (Unset/None/Active): returned unchanged.
if (ServiceResult.IsBad(result) || !_standby.DataProjected) return result;
value = Variant.Null;
sourceTimestamp = DateTimeUtc.MinValue;
return StatusCodes.BadOutOfService;
}
}
@@ -4,6 +4,7 @@ using Opc.Ua;
using Opc.Ua.Configuration;
using Opc.Ua.Identity;
using Opc.Ua.Server;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
@@ -174,6 +175,83 @@ public sealed class OpcUaApplicationHostOptions
/// Replaces the SDK's own lockout, which also refused Anonymous; see <see cref="FailedCredentialLockout"/>.
/// </summary>
public OpcUaCredentialLockoutOptions CredentialLockout { get; set; } = new();
/// <summary>
/// Warm non-transparent redundancy (<c>OpcUa:Redundancy:*</c>, Gitea #717). The default <c>Mode=Unset</c> is
/// byte-for-byte today's server. Startup-static.
/// </summary>
public OpcUaRedundancyOptions Redundancy { get; set; } = new();
}
/// <summary>The <c>OpcUa:Redundancy</c> section (Gitea #717, spec §6).</summary>
public sealed class OpcUaRedundancyOptions
{
/// <summary>
/// <c>Unset</c> (default) / <c>None</c> / <c>Warm</c>, case-insensitive. <c>Hot</c> is not advertisable
/// (spec §3) and, like any other value, fails host start. Bound as a string so a bad value is a readable
/// validation failure rather than a binder exception.
/// </summary>
public string Mode { get; set; } = nameof(OpcUaRedundancyMode.Unset);
/// <summary>
/// Warm only: let the survivor go Active while the Primary is unreachable but not yet downed (spec §4.1, owner
/// answer Q2). Default <c>true</c>.
/// </summary>
public bool PromoteWhilePrimaryUnreachable { get; set; } = true;
/// <summary>Parses <see cref="Mode"/>.</summary>
/// <param name="mode">The parsed mode.</param>
/// <returns><c>false</c> for <c>Hot</c>, numerics or any unknown value.</returns>
public bool TryParseMode(out OpcUaRedundancyMode mode)
{
var text = (Mode ?? string.Empty).Trim();
if (text.Length == 0) text = nameof(OpcUaRedundancyMode.Unset);
if (!text.All(char.IsLetter)) { mode = default; return false; }
return Enum.TryParse(text, ignoreCase: true, out mode) && Enum.IsDefined(mode);
}
/// <summary>The parsed mode; throws on an invalid one (the validator rejects it first).</summary>
public OpcUaRedundancyMode ParsedMode => TryParseMode(out var m)
? m
: throw new InvalidOperationException(
$"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable).");
/// <summary>The runtime settings the redundancy actors read.</summary>
/// <returns>The settings.</returns>
public WarmRedundancySettings ToSettings() => new(ParsedMode, PromoteWhilePrimaryUnreachable);
/// <summary>
/// The rules of spec §3 that need only configuration: <c>Warm</c> needs at least one peer, all URIs distinct
/// and the self URI not listed as a peer; <c>None</c> needs no peers. Returns the failure messages (empty when
/// valid). The effective self URI is checked again after an <c>ApplicationConfigPath</c> load.
/// </summary>
/// <param name="selfUri">The self URI to check against.</param>
/// <param name="peers">The configured peer URIs.</param>
/// <returns>The failures.</returns>
public IReadOnlyList<string> Check(string? selfUri, IEnumerable<string>? peers)
{
var failures = new List<string>();
if (!TryParseMode(out var mode))
{
failures.Add($"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable: continuous acquisition alone does not meet Part 4 Hot)");
return failures;
}
var list = (peers ?? Array.Empty<string>()).Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToList();
if (mode == OpcUaRedundancyMode.None && list.Count > 0)
failures.Add("OpcUa:PeerApplicationUris must be empty when OpcUa:Redundancy:Mode is None (a standalone server has no peers)");
if (mode == OpcUaRedundancyMode.Warm)
{
if (list.Count == 0)
failures.Add("OpcUa:Redundancy:Mode Warm requires at least one OpcUa:PeerApplicationUris entry");
if (list.Distinct(StringComparer.Ordinal).Count() != list.Count)
failures.Add("OpcUa:PeerApplicationUris must not contain duplicates under OpcUa:Redundancy:Mode Warm");
if (!string.IsNullOrWhiteSpace(selfUri) && list.Contains(selfUri.Trim(), StringComparer.Ordinal))
failures.Add($"OpcUa:PeerApplicationUris must not list this server's own ApplicationUri '{selfUri}' under OpcUa:Redundancy:Mode Warm (each pair node needs a unique ApplicationUri)");
}
return failures;
}
}
/// <summary>
@@ -289,7 +367,8 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
try
{
_ = await BuildConfigurationAsync(cancellationToken);
var configuration = await BuildConfigurationAsync(cancellationToken);
ConfigureRedundancy(sdkServer, configuration.ApplicationUri ?? _options.ApplicationUri);
await EnsureApplicationCertificateAsync(cancellationToken).ConfigureAwait(false);
AttachUserAuthenticator(sdkServer);
await _application.StartAsync(server, cancellationToken).ConfigureAwait(false);
@@ -305,6 +384,31 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
_options.PublicHostname, _options.OpcUaPort);
}
/// <summary>
/// Gitea #717 (spec §3): validate the redundancy mode against the EFFECTIVE self URI — the loaded
/// <c>ApplicationConfigPath</c> XML wins over <c>OpcUa:ApplicationUri</c> (c7787) — log the boot banner and hand
/// the startup-static metadata to the SDK server, which installs it in <c>OnNodeManagerStartedAsync</c>.
/// <c>Unset</c> hands nothing over, so the SDK's graph is untouched.
/// </summary>
/// <param name="server">The SDK server about to start.</param>
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
/// <exception cref="InvalidOperationException">The redundancy configuration is invalid for the effective URI.</exception>
private void ConfigureRedundancy(OtOpcUaSdkServer server, string effectiveSelfUri)
{
var redundancy = _options.Redundancy ?? new OpcUaRedundancyOptions();
var failures = redundancy.Check(effectiveSelfUri, _options.PeerApplicationUris);
if (failures.Count > 0)
throw new InvalidOperationException("OPC UA redundancy configuration is invalid: " + string.Join("; ", failures));
var mode = redundancy.ParsedMode;
var peers = (_options.PeerApplicationUris ?? new List<string>())
.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToArray();
_logger.LogInformation(
"OPC UA redundancy: Mode={Mode}, effective ApplicationUri={SelfUri}, peers=[{Peers}], PromoteWhilePrimaryUnreachable={Promote}",
mode, effectiveSelfUri, string.Join(", ", peers), redundancy.PromoteWhilePrimaryUnreachable);
server.ConfigureRedundancy(mode, effectiveSelfUri, peers);
}
private void AttachUserAuthenticator(OtOpcUaSdkServer server)
{
foreach (var type in new[] { UserTokenType.Anonymous, UserTokenType.UserName })
@@ -161,6 +161,34 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
private FolderState? _root;
private readonly ILogger<OtOpcUaNodeManager> _logger;
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
/// <param name="server">The OPC UA server instance.</param>
/// <param name="configuration">The application configuration.</param>
/// <param name="logger">Logger for the manager's swallowed-fault and skipped-wiring diagnostics;
/// <c>null</c> (tests) falls back to <see cref="NullLogger{T}.Instance"/>.</param>
/// <param name="standby">The Warm standby flags (Gitea #717); null (every non-Warm server) is
/// <see cref="WarmStandbyState.Off"/>, which leaves every data path exactly as it was.</param>
internal OtOpcUaNodeManager(
IServerInternal server, ApplicationConfiguration configuration, ILogger<OtOpcUaNodeManager>? logger,
WarmStandbyState? standby)
: this(server, configuration, logger)
{
_standby = standby ?? WarmStandbyState.Off;
if (_standby.WarmMode)
{
// Spec §4.6 item 1: ordinary-tag Value items are built by OrdinaryTagMonitoredItem, which marks each
// notification. Warm only — Unset/None keep the SDK's own manager.
m_monitoredItemManager.Dispose();
m_monitoredItemManager = new OrdinaryTagItemManager(this.ToAsyncNodeManager(), server);
}
}
/// <summary>The Warm standby flags (Gitea #717). <see cref="WarmStandbyState.Off"/> unless the server is Warm.</summary>
private readonly WarmStandbyState _standby = WarmStandbyState.Off;
/// <summary>Test observation: the Warm standby flags.</summary>
internal WarmStandbyState Standby => _standby;
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
/// <param name="server">The OPC UA server instance.</param>
/// <param name="configuration">The application configuration.</param>
@@ -1434,6 +1462,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
// the InstanceStateSnapshot captures the just-projected values.
alarm.ClearChangeMasks(SystemContext, includeChildren: true);
// Gitea #717 spec §4.5: on a Warm Standby only the STAMP half above runs — the condition node is current and a
// later refresh carries a valid, ackable EventId — and the DELIVER half below does not. Suppressing here, before
// the snapshot and every SDK field projection, covers the Server-object, folder and direct paths, ACL on or
// off, by construction; overflow, filtered retain and the #838 deferral queue never see the event. The
// decision is made at report time: an event reported while Active is delivered even if that ends after
// demotion.
if (_standby.AlarmsSuppressed)
{
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbySuppressedAlarmEvents.Add(1);
return;
}
try
{
// InstanceStateSnapshot is the IFilterTarget — a frozen copy of the condition's fields at fire
@@ -2991,6 +3031,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
var gate = EvaluateAlarmCommandGate(identity, condition, operation);
if (gate is not null) return gate;
// Gitea #717 owner answer Q4: a Warm Standby refuses every Part 9 condition method before it changes any state
// (nothing routed, nothing applied, no event). Unset/None keep today's acceptance.
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal($"call {operation}");
var cmd = new AlarmCommand(
AlarmId: condition.NodeId.IdentifierAsString,
Operation: operation,
@@ -3032,6 +3076,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
var gate = EvaluateAlarmCommandGate(identity, condition, "Acknowledge");
if (gate is not null) return gate;
// Gitea #717 owner answer Q4: refused on a Warm Standby, ahead of (and whatever) the Primary gate below says.
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("acknowledge the alarm");
// PRIMARY GATE, after authorization (a caller who may not ack learns nothing about the
// node's role). Only the Primary routes a native ack upstream (DriverHostActor applies the same verdict
// to the route), so on any other node the ack is refused outright rather than applied locally and then
@@ -3215,6 +3262,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
?? EvaluateEquipmentWriteGate(identity, gatewayWired);
if (gate is not null) return gate;
// Gitea #717 owner answer Q1: a Warm Standby refuses the write synchronously, AFTER authorization (a caller who
// may not write learns nothing about the node's role) and before anything reaches the driver. Nothing is
// applied, so nothing is reverted. Unset/None never set the flag and keep the #5 optimistic path.
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("write the value");
// Item A (synchronous structural fail-fast): reject a structurally-invalid write INLINE — return Bad
// synchronously so the SDK never applies it (no optimistic-Good-then-revert round-trip + no needless
// device dispatch). Runs AFTER the authz gate (so we never leak structure detail to an unauthorised
@@ -4120,7 +4172,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
// The SDK exposes the flags separately (no CurrentReadWrite composite): ReadWrite is
// CurrentRead | CurrentWrite, with the HistoryRead bit OR-ed in when historized.
var access = ComposeAccessLevel(writable, historized, HistoryWritable(_historianDataUpdater));
var variable = new EquipmentVariableState(parent, Lock)
var variable = new EquipmentVariableState(parent, Lock, _standby)
{
NodeId = new NodeId(variableNodeId, ns),
BrowseName = new QualifiedName(variableNodeId, ns),
@@ -5725,6 +5777,88 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
lock (Lock) return MonitoredNodes.ContainsKey(nodeId);
}
/// <summary>
/// Gitea #717 spec §4.7 promotion step 2 / demotion step 5: queues one value to every live, non-stood-down
/// ordinary-tag <c>Value</c> item, ignoring its filters (the queue pattern of <see cref="RevalidateMonitoredItems"/>
/// — never detach or <c>Disabled</c>, which drops the queue). Promotion queues the current stored value read
/// through the item's own IndexRange/DataEncoding; demotion queues <c>BadOutOfService</c>/null.
/// </summary>
/// <param name="projected">Queue the projected value (demotion) instead of the stored one (promotion).</param>
/// <returns>The number of items queued to.</returns>
internal int QueueCurrentToLiveItems(bool projected)
{
var queued = 0;
foreach (var (item, tracked) in _liveItems)
{
if (tracked.Suppressed || tracked.Handle.Node is not EquipmentVariableState variable) continue;
if (item is not IDataChangeMonitoredItem2 { AttributeId: Attributes.Value } data) continue;
var now = UtcNow;
DataValue value;
if (projected)
{
value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, now);
}
else
{
var stored = new DataValue();
ServiceResult read;
lock (Lock) read = variable.ReadAttribute(SystemContext, Attributes.Value, data.IndexRange, data.DataEncoding, ref stored);
value = ServiceResult.IsBad(read)
? new DataValue(Variant.Null, read.StatusCode, DateTimeUtc.MinValue, now)
: new DataValue(stored.WrappedValue, stored.StatusCode, stored.SourceTimestamp, now);
}
try
{
item.QueueValue(value, StatusCode.IsBad(value.StatusCode) ? new ServiceResult(value.StatusCode) : null, ignoreFilters: true);
queued++;
}
catch (Exception ex)
{
// A disabled item refuses the value; it re-samples the (projected or not) node when re-enabled.
_logger.LogDebug(ex, "OtOpcUaNodeManager: could not queue a Warm transition value to a monitored item");
}
}
return queued;
}
/// <summary>
/// Gitea #717 spec §4.5 promotion refresh: a ConditionRefresh2 on every live event item — Server-object items
/// and the direct/folder items of this manager's notifiers — through the SDK's own subscription path, so each
/// client sees RefreshStart, the retained conditions (through this manager's ACL decision and the #854/#855
/// cut), then RefreshEnd. Must not be called under <c>Lock</c>.
/// </summary>
/// <returns>The number of event items refreshed.</returns>
internal int RefreshAllEventItems()
{
var items = new List<IEventMonitoredItem>(_allEventsItems.Keys);
foreach (var (item, tracked) in _eventItems)
if (!tracked.Suppressed) items.Add(item);
if (items.Count == 0) return 0;
var subscriptions = new Dictionary<uint, ISubscription>();
foreach (var subscription in Server.SubscriptionManager.GetSubscriptions()) subscriptions[subscription.Id] = subscription;
var refreshed = 0;
foreach (var item in items)
{
if (!subscriptions.TryGetValue(item.SubscriptionId, out var subscription)) continue;
try
{
subscription.ConditionRefresh2Async(item.Id).AsTask().GetAwaiter().GetResult();
refreshed++;
}
catch (Exception ex)
{
// A subscription deleted mid-walk, or an item whose mode refuses a refresh: that client re-refreshes itself.
_logger.LogDebug(ex, "OtOpcUaNodeManager: Warm promotion refresh of event item {ItemId} failed", item.Id);
}
}
return refreshed;
}
/// <summary>Test observability: how many data-change items the revalidation map tracks.</summary>
internal int LiveItemCount => _liveItems.Count;
@@ -315,6 +315,160 @@ public sealed class OtOpcUaSdkServer : StandardServer
/// </summary>
public const byte StartupServiceLevel = 0;
private OpcUaRedundancyMode _redundancyMode = OpcUaRedundancyMode.Unset;
private string _selfUri = string.Empty;
private string[] _peerUris = [];
private WarmStandbyState _standby = WarmStandbyState.Off;
/// <summary>The redundancy mode configured before start (Gitea #717).</summary>
public OpcUaRedundancyMode RedundancyMode => _redundancyMode;
/// <summary>The Warm standby flags shared with the node manager.</summary>
internal WarmStandbyState Standby => _standby;
/// <summary>The initial <c>Server.ServiceLevel</c> under <c>Warm</c> (spec §4.3: the NoData band, replacing the
/// SDK's 255 and this server's #885 value 0 until the first computed level).</summary>
public const byte WarmBootServiceLevel = 1;
/// <summary>
/// Sets the startup-static redundancy metadata (Gitea #717 spec §3) before <c>StartAsync</c>.
/// <see cref="OpcUaRedundancyMode.Unset"/> (the default when this is never called) leaves the SDK graph and
/// every data path exactly as they are today.
/// </summary>
/// <param name="mode">The validated mode.</param>
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
/// <param name="peerUris">The peer URIs, in configured order.</param>
public void ConfigureRedundancy(OpcUaRedundancyMode mode, string effectiveSelfUri, IReadOnlyList<string> peerUris)
{
if (_otOpcUaNodeManager is not null)
throw new InvalidOperationException("Redundancy must be configured before the server starts.");
_redundancyMode = mode;
_selfUri = effectiveSelfUri;
_peerUris = peerUris.ToArray();
_standby = new WarmStandbyState(mode == OpcUaRedundancyMode.Warm);
}
/// <summary>
/// The data-plane transition steps a Warm <c>OpcUaPublishActor</c> drives (spec §4.7), or null when the server
/// is not Warm or has not started.
/// </summary>
/// <returns>The projection, or null.</returns>
public IStandbyProjection? CreateStandbyProjection() =>
_standby.WarmMode && _otOpcUaNodeManager is { } nm ? new SdkStandbyProjection(nm, _standby) : null;
/// <summary>Test seam: ids of the live subscriptions, for the decision record's dead-subscription sweep.</summary>
private IReadOnlySet<uint> LiveSubscriptionIds()
{
var ids = new HashSet<uint>();
try
{
if (CurrentInstance?.SubscriptionManager is { } manager)
foreach (var subscription in manager.GetSubscriptions()) ids.Add(subscription.Id);
}
catch (Exception)
{
// Shutdown race: an empty set only drops records, which re-decide (fail-safe) at their next delivery.
}
return ids;
}
private long _publishesSinceSweep;
/// <inheritdoc />
public override async ValueTask<PublishResponse> PublishAsync(SecureChannelContext secureChannelContext,
RequestHeader? requestHeader, ArrayOf<SubscriptionAcknowledgement> subscriptionAcknowledgements,
RequestLifetime requestLifetime)
{
var response = await base.PublishAsync(secureChannelContext, requestHeader, subscriptionAcknowledgements, requestLifetime)
.ConfigureAwait(false);
if (_standby.Delivery is not { } record || response?.NotificationMessage is not { } message) return response!;
if (message.NotificationData.Count > 0)
{
response.NotificationMessage = Decide(record, response.SubscriptionId, message);
var available = new List<uint>(response.AvailableSequenceNumbers.Count);
for (var i = 0; i < response.AvailableSequenceNumbers.Count; i++) available.Add(response.AvailableSequenceNumbers[i]);
record.Prune(response.SubscriptionId, available, message.SequenceNumber);
}
if (Interlocked.Increment(ref _publishesSinceSweep) % 1024 == 0) record.SweepDead(LiveSubscriptionIds());
return response;
}
/// <inheritdoc />
public override async ValueTask<RepublishResponse> RepublishAsync(SecureChannelContext secureChannelContext,
RequestHeader? requestHeader, uint subscriptionId, uint retransmitSequenceNumber, RequestLifetime requestLifetime)
{
var response = await base.RepublishAsync(secureChannelContext, requestHeader, subscriptionId, retransmitSequenceNumber,
requestLifetime).ConfigureAwait(false);
if (_standby.Delivery is { } record && response?.NotificationMessage is { NotificationData.Count: > 0 } message)
response.NotificationMessage = Decide(record, subscriptionId, message);
return response!;
}
/// <summary>The recorded (or, first time, current) decision for one delivery, applied to the SDK's fresh clone.</summary>
private NotificationMessage Decide(FirstDeliveryRecord record, uint subscriptionId, NotificationMessage message)
{
if (!record.Decide(subscriptionId, message.SequenceNumber)) return message;
var projected = StandbyNotificationRewriter.Project(message, out var rewritten);
if (rewritten > 0)
{
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbyProjectedNotifications.Add(rewritten);
}
return projected;
}
/// <summary>
/// Installs the standard redundancy metadata (Gitea #717 spec §3, T0a): under <c>Warm</c>
/// <c>Server.ServerRedundancy</c> becomes a <c>NonTransparentRedundancyType</c> whose read-only
/// <c>ServerUriArray</c> (<c>i=11314</c>) is <c>[self, peers…]</c>, <c>RedundancySupport</c> is <c>Warm</c> and
/// ServiceLevel boots at <see cref="WarmBootServiceLevel"/>; under <c>None</c> <c>RedundancySupport</c> is
/// <c>None</c> and the SDK's orphan <c>i=11314</c> is deleted, so it is absent by NodeId as well as by browse.
/// <c>Unset</c> touches nothing.
/// </summary>
private async ValueTask InstallRedundancyMetadataAsync(IServerInternal server, CancellationToken cancellationToken)
{
if (_redundancyMode == OpcUaRedundancyMode.Unset) return;
var serverObject = server.ServerObject
?? throw new InvalidOperationException("The Server object is unavailable during server startup.");
var context = server.DefaultSystemContext;
var redundancy = serverObject.ServerRedundancy
?? throw new InvalidOperationException("Server.ServerRedundancy is unavailable during server startup.");
if (_redundancyMode == OpcUaRedundancyMode.None)
{
redundancy.RedundancySupport!.Value = RedundancySupport.None;
redundancy.RedundancySupport.ClearChangeMasks(context, false);
// T0a finding: the SDK's predefined i=11314 survives as an orphan (unreferenced, but Good [] by NodeId).
if (server.DiagnosticsNodeManager is AsyncCustomNodeManager diagnostics)
await diagnostics.DeleteNodeAsync(context, VariableIds.Server_ServerRedundancy_ServerUriArray, cancellationToken)
.ConfigureAwait(false);
return;
}
var replacer = server.DiagnosticsNodeManager as IPredefinedNodeSubtypeReplacer
?? throw new InvalidOperationException("The diagnostics node manager cannot replace predefined node subtypes.");
var replaced = (NonTransparentRedundancyState)await replacer.ReplacePredefinedInstanceSubtypeAsync(context, redundancy,
context.CreateInstanceOfNonTransparentRedundancyType(),
new Dictionary<QualifiedName, NodeId>
{
[new QualifiedName(BrowseNames.ServerUriArray, 0)] = VariableIds.Server_ServerRedundancy_ServerUriArray,
},
node => serverObject.ServerRedundancy = (ServerRedundancyState)node, cancellationToken).ConfigureAwait(false);
replaced.RedundancySupport!.Value = RedundancySupport.Warm;
replaced.ServerUriArray!.Value = (ArrayOf<string>)new[] { _selfUri }.Concat(_peerUris).ToArray();
replaced.ServerUriArray.AccessLevel = replaced.ServerUriArray.UserAccessLevel = AccessLevels.CurrentRead;
replaced.ClearChangeMasks(context, true);
if (serverObject.ServiceLevel is { } serviceLevel)
{
serviceLevel.Value = WarmBootServiceLevel;
serviceLevel.Timestamp = DateTime.UtcNow;
serviceLevel.ClearChangeMasks(context, includeChildren: false);
}
}
/// <inheritdoc />
protected override async ValueTask OnNodeManagerStartedAsync(
IServerInternal server, CancellationToken cancellationToken = default)
@@ -329,6 +483,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
serviceLevel.Timestamp = DateTime.UtcNow;
serviceLevel.ClearChangeMasks(server.DefaultSystemContext, includeChildren: false);
}
await InstallRedundancyMetadataAsync(server, cancellationToken).ConfigureAwait(false);
var auditType = server.DiagnosticsNodeManager?
.FindPredefinedNode<BaseObjectTypeState>(ObjectTypeIds.AuditWriteUpdateEventType)
?? throw new InvalidOperationException("AuditWriteUpdateEventType is unavailable during server startup.");
@@ -360,7 +515,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
protected override MasterNodeManager CreateMasterNodeManager(
IServerInternal server, ApplicationConfiguration configuration)
{
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger);
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger, _standby);
return new MasterNodeManager(server, configuration, dynamicNamespaceUri: null, _otOpcUaNodeManager);
}
}
@@ -0,0 +1,388 @@
using Microsoft.Extensions.Logging;
using Opc.Ua;
using Opc.Ua.Server;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <summary>
/// The node-manager-wide Warm standby flags (Gitea #717, spec §4.4–§4.6). One instance per server. Under
/// <c>Unset</c>/<c>None</c> <see cref="WarmMode"/> is false and nothing ever sets a flag, so every reader sees
/// today's behaviour. Under <c>Warm</c> the node starts Standby (all flags set) until the redundancy layer
/// promotes it.
/// </summary>
internal sealed class WarmStandbyState
{
private volatile bool _dataProjected;
private volatile bool _alarmsSuppressed;
/// <summary>Initializes the state.</summary>
/// <param name="warmMode">Whether this server runs the Warm role contract.</param>
public WarmStandbyState(bool warmMode)
{
WarmMode = warmMode;
_dataProjected = warmMode;
_alarmsSuppressed = warmMode;
Delivery = warmMode ? new FirstDeliveryRecord(standby: true) : null;
}
/// <summary>The not-Warm state (no projection, no suppression, no decision record).</summary>
public static WarmStandbyState Off { get; } = new(false);
/// <summary>Whether this server runs the Warm role contract.</summary>
public bool WarmMode { get; }
/// <summary>Ordinary-tag reads/samples project <c>BadOutOfService</c>; writes and Part 9 methods are refused.</summary>
public bool DataProjected
{
get => _dataProjected;
set => _dataProjected = WarmMode && value;
}
/// <summary>New alarm condition events are not delivered (still stamped).</summary>
public bool AlarmsSuppressed
{
get => _alarmsSuppressed;
set => _alarmsSuppressed = WarmMode && value;
}
/// <summary>Whether a Warm Standby must refuse a write or a Part 9 method (owner answers Q1/Q4).</summary>
public bool RefusesMutations => WarmMode && _dataProjected;
/// <summary>The first-delivery decision record; null when not Warm.</summary>
public FirstDeliveryRecord? Delivery { get; }
/// <summary>The refusal returned to a write or a Part 9 method on a Warm Standby.</summary>
public static ServiceResult StandbyRefusal(string what) => new(StatusCodes.BadInvalidState,
new LocalizedText($"This server is the Warm redundancy standby; {what} on the active server (the one with the higher ServiceLevel)."));
}
/// <summary>
/// The ordinary-tag marker (spec §4.6 item 1): set on <see cref="MonitoredItemNotification.Message"/>, a helper
/// reference the binary encoder never writes and the SDK's per-delivery <c>CoreUtils.Clone</c> copies
/// (pinned by <c>WarmStandbySdkFieldContractTests</c>). Valid only for the notification whose
/// <see cref="ClientHandle"/> and <see cref="Value"/> it captured, so a stale marker left on a pooled payload
/// (2.0.1's <c>Initialize()</c> does not clear the helper) fails <see cref="Matches"/>.
/// </summary>
/// <remarks>Derives from <see cref="NotificationMessage"/> only because that is the helper's declared type.</remarks>
internal sealed class OrdinaryTagMarker : NotificationMessage
{
/// <summary>Initializes the marker.</summary>
/// <param name="clientHandle">The notification's client handle when it was built.</param>
/// <param name="value">The notification's final value when it was built.</param>
public OrdinaryTagMarker(uint clientHandle, DataValue value)
{
ClientHandle = clientHandle;
Value = value;
}
/// <summary>The client handle captured at build time.</summary>
public uint ClientHandle { get; }
/// <summary>The value captured at build time.</summary>
public DataValue Value { get; }
/// <summary>Whether <paramref name="n"/> is the notification this marker was made for.</summary>
/// <param name="n">A delivered notification.</param>
/// <returns>True for a match on client handle and value.</returns>
public bool Matches(MonitoredItemNotification n) => n.ClientHandle == ClientHandle && Equals(n.Value, Value);
/// <summary>Whether <paramref name="n"/> carries a valid ordinary-tag marker.</summary>
/// <param name="n">A delivered notification.</param>
/// <returns>True when it is an ordinary-tag notification.</returns>
public static bool IsOrdinaryTag(MonitoredItemNotification n) => n.Message is OrdinaryTagMarker m && m.Matches(n);
}
/// <summary>
/// The per-message first-delivery decision (spec §4.6 item 2, D4), keyed by
/// <c>(SubscriptionId, SequenceNumber)</c> — never by object: SDK 2.0.1 hands every Publish and Republish a fresh
/// clone of the cached message. The first delivery of a sequence number records Active or Standby; every later
/// delivery (Republish included) applies the recorded decision. One lock orders a decision against a role flip;
/// it is never held across an await or a projection.
/// </summary>
internal sealed class FirstDeliveryRecord
{
/// <summary>Recorded messages kept per subscription beyond what <c>AvailableSequenceNumbers</c> prunes.</summary>
internal const int MaxPerSubscription = 1024;
/// <summary>Subscriptions kept before the least-recently-used one is dropped (dead subscriptions).</summary>
internal const int MaxSubscriptions = 8192;
private readonly object _gate = new();
private readonly Dictionary<uint, SubscriptionRecord> _subscriptions = new();
private bool _standby;
private long _touch;
/// <summary>Initializes the record.</summary>
/// <param name="standby">The initial delivery state (a Warm node boots Standby).</param>
public FirstDeliveryRecord(bool standby) => _standby = standby;
/// <summary>Test seam: runs inside the lock after the decision is read and before it is recorded.</summary>
internal Action? DecisionBarrier { get; set; }
/// <summary>The current first-delivery state.</summary>
public bool Standby
{
get { lock (_gate) return _standby; }
}
/// <summary>Flips the first-delivery state (the D4 boundary). Waits for a decision in progress.</summary>
/// <param name="standby">Whether later first deliveries are Standby.</param>
public void SetStandby(bool standby)
{
lock (_gate) _standby = standby;
}
/// <summary>The decision for one delivery: the recorded one, or (first delivery) the current state, recorded.</summary>
/// <param name="subscriptionId">The subscription.</param>
/// <param name="sequenceNumber">The message's sequence number.</param>
/// <returns>True when the delivery must be projected (Standby).</returns>
public bool Decide(uint subscriptionId, uint sequenceNumber)
{
lock (_gate)
{
if (!_subscriptions.TryGetValue(subscriptionId, out var sub))
{
if (_subscriptions.Count >= MaxSubscriptions) EvictLeastRecentlyUsed();
_subscriptions[subscriptionId] = sub = new SubscriptionRecord();
}
sub.Touched = ++_touch;
if (sub.Decisions.TryGetValue(sequenceNumber, out var recorded)) return recorded;
var standby = _standby;
DecisionBarrier?.Invoke();
sub.Decisions[sequenceNumber] = standby;
sub.Order.Enqueue(sequenceNumber);
while (sub.Order.Count > MaxPerSubscription && sub.Order.TryDequeue(out var oldest)) sub.Decisions.Remove(oldest);
return standby;
}
}
/// <summary>Drops the decisions of messages the subscription no longer retains (they can never be republished).</summary>
/// <param name="subscriptionId">The subscription the Publish response belongs to.</param>
/// <param name="available">The response's <c>AvailableSequenceNumbers</c>.</param>
/// <param name="justDelivered">The sequence number just delivered (kept: it becomes available for Republish).</param>
public void Prune(uint subscriptionId, IReadOnlyCollection<uint> available, uint justDelivered)
{
lock (_gate)
{
if (!_subscriptions.TryGetValue(subscriptionId, out var sub)) return;
var keep = new HashSet<uint>(available) { justDelivered };
var kept = new Queue<uint>();
while (sub.Order.TryDequeue(out var seq))
{
if (keep.Contains(seq)) kept.Enqueue(seq);
else sub.Decisions.Remove(seq);
}
sub.Order.Clear();
foreach (var seq in kept) sub.Order.Enqueue(seq);
}
}
/// <summary>Drops every record of subscriptions not in <paramref name="live"/>.</summary>
/// <param name="live">The ids of live subscriptions.</param>
public void SweepDead(IReadOnlySet<uint> live)
{
lock (_gate)
{
foreach (var id in _subscriptions.Keys.Where(id => !live.Contains(id)).ToList()) _subscriptions.Remove(id);
}
}
/// <summary>Recorded decisions (test observation).</summary>
internal int Count
{
get { lock (_gate) return _subscriptions.Values.Sum(s => s.Decisions.Count); }
}
/// <summary>Recorded subscriptions (test observation).</summary>
internal int SubscriptionCount
{
get { lock (_gate) return _subscriptions.Count; }
}
private void EvictLeastRecentlyUsed()
{
var victim = _subscriptions.MinBy(kv => kv.Value.Touched).Key;
_subscriptions.Remove(victim);
}
private sealed class SubscriptionRecord
{
public readonly Dictionary<uint, bool> Decisions = new();
public readonly Queue<uint> Order = new();
public long Touched;
}
}
/// <summary>
/// Rewrites a Standby delivery (spec §4.6): every marked ordinary-tag notification whose status is not Bad becomes
/// <c>BadOutOfService</c>/null with its server timestamp kept. Builds new objects; the message handed in (the SDK's
/// fresh per-delivery clone) and, a fortiori, the SDK's cached copy are never mutated.
/// </summary>
internal static class StandbyNotificationRewriter
{
/// <summary>Projects <paramref name="message"/>, or returns it unchanged when nothing needs rewriting.</summary>
/// <param name="message">A delivered message.</param>
/// <param name="rewritten">The number of notifications rewritten.</param>
/// <returns>The message to deliver.</returns>
public static NotificationMessage Project(NotificationMessage message, out int rewritten)
{
rewritten = 0;
if (message is null || message.NotificationData.Count == 0) return message!;
var data = new List<ExtensionObject>(message.NotificationData.Count);
var changed = false;
for (var i = 0; i < message.NotificationData.Count; i++)
{
var eo = message.NotificationData[i];
if (eo.TryGetValue<DataChangeNotification>(out var dcn) && dcn is not null)
{
List<MonitoredItemNotification>? items = null;
for (var j = 0; j < dcn.MonitoredItems.Count; j++)
{
var n = dcn.MonitoredItems[j];
if (OrdinaryTagMarker.IsOrdinaryTag(n) && !StatusCode.IsBad(n.Value.StatusCode))
{
items ??= CopyPrefix(dcn, j);
items.Add(new MonitoredItemNotification
{
ClientHandle = n.ClientHandle,
Value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, n.Value.ServerTimestamp),
});
rewritten++;
}
else
{
items?.Add(n);
}
}
if (items is not null)
{
changed = true;
data.Add(new ExtensionObject(new DataChangeNotification { MonitoredItems = items, DiagnosticInfos = dcn.DiagnosticInfos }));
continue;
}
}
data.Add(eo);
}
if (!changed) return message;
return new NotificationMessage { SequenceNumber = message.SequenceNumber, PublishTime = message.PublishTime, NotificationData = data };
}
private static List<MonitoredItemNotification> CopyPrefix(DataChangeNotification dcn, int count)
{
var list = new List<MonitoredItemNotification>(dcn.MonitoredItems.Count);
for (var k = 0; k < count; k++) list.Add(dcn.MonitoredItems[k]);
return list;
}
}
/// <summary>
/// A sampled data-change item for an ordinary-tag <c>Value</c> (spec §4.6 item 1): after the SDK's own per-value
/// publish (so the timestamp filter has run) it marks each notification it produced with an
/// <see cref="OrdinaryTagMarker"/>. Everything else is the SDK's <see cref="MonitoredItem"/>.
/// </summary>
internal sealed class OrdinaryTagMonitoredItem : MonitoredItem
{
/// <summary>Initializes the item exactly as <see cref="MonitoredNodeMonitoredItemManager"/> does.</summary>
public OrdinaryTagMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager, NodeHandle handle, uint subscriptionId,
uint id, MonitoredItemCreateRequest request, DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn,
MonitoringFilter? filter, Opc.Ua.Range? range, double samplingInterval, uint queueSize, bool durable)
: base(server, nodeManager, handle, subscriptionId, id, request.ItemToMonitor, diagnosticsMasks, timestampsToReturn,
request.MonitoringMode, request.RequestedParameters.ClientHandle, filter, filter, range, samplingInterval, queueSize,
request.RequestedParameters.DiscardOldest, 0, durable)
{
}
/// <inheritdoc />
public override bool Publish(OperationContext context, Queue<MonitoredItemNotification> notifications,
Queue<DiagnosticInfo> diagnostics, uint maxNotificationsPerPublish, ILogger logger)
{
var local = new Queue<MonitoredItemNotification>();
var localDiagnostics = new Queue<DiagnosticInfo>();
var more = base.Publish(context, local, localDiagnostics, maxNotificationsPerPublish, logger);
while (local.Count > 0)
{
var n = local.Dequeue();
n.Message = new OrdinaryTagMarker(n.ClientHandle, n.Value);
notifications.Enqueue(n);
if (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
}
while (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
return more;
}
}
/// <summary>
/// Installs <see cref="OrdinaryTagMonitoredItem"/> for ordinary-tag (<see cref="EquipmentVariableState"/>)
/// <c>Value</c> items only. Re-implements <see cref="IMonitoredItemManager.CreateMonitoredItem"/> on the SDK's
/// public, non-sealed manager, so <c>CustomNodeManager2</c>'s <c>is MonitoredNodeMonitoredItemManager</c> checks and
/// its internal lifecycle interface keep working (T0c; the 2.0.1 <c>MonitoredItemFactory</c> hook exists only on
/// <c>AsyncCustomNodeManager</c>). Installed only under Warm; <c>Unset</c>/<c>None</c> keep the SDK's manager.
/// </summary>
internal sealed class OrdinaryTagItemManager(IAsyncNodeManager nodeManager, IServerInternal server)
: MonitoredNodeMonitoredItemManager(nodeManager, server), IMonitoredItemManager
{
/// <inheritdoc />
ISampledDataChangeMonitoredItem IMonitoredItemManager.CreateMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager,
ServerSystemContext context, NodeHandle handle, uint subscriptionId, double publishingInterval,
DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn, MonitoredItemCreateRequest itemToCreate,
Opc.Ua.Range euRange, MonitoringFilter filterToUse, double samplingInterval, uint revisedQueueSize, bool createDurable,
MonitoredItemIdFactory monitoredItemIdFactory, Func<ISystemContext, NodeHandle, NodeState, NodeState> addNodeToComponentCache)
{
if (handle.Node is not EquipmentVariableState || itemToCreate.ItemToMonitor.AttributeId != Attributes.Value)
{
return CreateMonitoredItem(server, nodeManager, context, handle, subscriptionId, publishingInterval, diagnosticsMasks,
timestampsToReturn, itemToCreate, euRange, filterToUse, samplingInterval, revisedQueueSize, createDurable,
monitoredItemIdFactory, addNodeToComponentCache);
}
// Mirrors MonitoredNodeMonitoredItemManager.CreateMonitoredItem in exact tagged 2.0.1, swapping only the item type.
var cachedNode = addNodeToComponentCache(context, handle, handle.Node);
if (!MonitoredNodes.TryGetValue(handle.Node.NodeId, out var monitoredNode))
{
MonitoredNodes[handle.Node.NodeId] = monitoredNode = new MonitoredNode2(nodeManager, server, cachedNode,
nodeManager.IsMultipleEventConsumerNode(cachedNode.NodeId));
}
handle.Node = monitoredNode.Node;
handle.MonitoredNode = monitoredNode;
uint id;
do
{
id = monitoredItemIdFactory.GetNextId();
}
while (!MonitoredItems.TryAdd(id, null!));
var item = new OrdinaryTagMonitoredItem(server, nodeManager, handle, subscriptionId, id, itemToCreate, diagnosticsMasks,
timestampsToReturn, filterToUse, euRange, samplingInterval, revisedQueueSize, createDurable);
monitoredNode.Add((IDataChangeMonitoredItem2)item);
MonitoredItems[id] = item;
return item;
}
}
/// <summary>The <see cref="ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection"/> a Warm server hands the redundancy layer.</summary>
internal sealed class SdkStandbyProjection(OtOpcUaNodeManager nodeManager, WarmStandbyState state)
: ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection
{
/// <inheritdoc />
public void SetDataProjection(bool projected) => state.DataProjected = projected;
/// <inheritdoc />
public void SetStandbyDelivery(bool standby) => state.Delivery?.SetStandby(standby);
/// <inheritdoc />
public void SetAlarmSuppression(bool suppressed) => state.AlarmsSuppressed = suppressed;
/// <inheritdoc />
public int QueueCurrentToLiveItems(bool projected) => nodeManager.QueueCurrentToLiveItems(projected);
/// <inheritdoc />
public int RefreshAllEventItems() => nodeManager.RefreshAllEventItems();
}
@@ -258,6 +258,52 @@ public static class DeploymentArtifact
}
}
/// <summary>
/// Gitea #717 spec §3: the <c>ServerCluster.RedundancyMode</c> of this node's cluster as the artifact carries it
/// (<c>None</c>/<c>Warm</c>/<c>Hot</c>), for the deploy-time cross-check against <c>OpcUa:Redundancy:Mode</c>.
/// A single-cluster artifact answers for its only cluster; a multi-cluster one for the cluster of this node's
/// <c>ClusterNode</c> row. Null when the artifact does not say (empty, malformed, no matching cluster).
/// </summary>
/// <param name="blob">The deployment artifact.</param>
/// <param name="nodeId">This node's <c>host:port</c> id.</param>
/// <returns>The mode name, or null.</returns>
public static string? ReadClusterRedundancyMode(ReadOnlySpan<byte> blob, string nodeId)
{
if (blob.IsEmpty) return null;
try
{
using var parsed = Open(blob.ToArray());
var root = parsed.Root;
if (!root.TryGetProperty("Clusters", out var clusters) || clusters.ValueKind != JsonValueKind.Array) return null;
var scope = parsed.ResolveClusterScope(nodeId);
JsonElement? match = null;
foreach (var cluster in clusters.EnumerateArray())
{
if (cluster.ValueKind != JsonValueKind.Object) continue;
if (scope.Mode == ClusterFilterMode.None) { match = cluster; break; }
if (scope.Mode == ClusterFilterMode.ScopeTo
&& string.Equals(cluster.TryGetProperty("ClusterId", out var cid) && cid.ValueKind == JsonValueKind.String ? cid.GetString() : null, scope.ClusterId, StringComparison.OrdinalIgnoreCase))
{
match = cluster;
break;
}
}
if (match is not { } m || !m.TryGetProperty("RedundancyMode", out var mode)) return null;
return mode.ValueKind switch
{
JsonValueKind.String => mode.GetString(),
JsonValueKind.Number when mode.TryGetInt32(out var n) && Enum.IsDefined(typeof(Configuration.Enums.RedundancyMode), n)
=> ((Configuration.Enums.RedundancyMode)n).ToString(),
_ => null,
};
}
catch (JsonException)
{
return null;
}
}
/// <summary>
/// Resolve how a node should scope a deployment artifact to its own ClusterId. Single-cluster
/// (or legacy) artifacts resolve to <see cref="ClusterFilterMode.None"/> so every existing
@@ -789,7 +789,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null) =>
TimeSpan? childShutdownTimeout = null,
bool enforceSnapshotOrdering = false) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -803,7 +804,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout));
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout,
enforceSnapshotOrdering));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -938,8 +940,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null)
TimeSpan? childShutdownTimeout = null,
bool enforceSnapshotOrdering = false)
{
_snapshotOrdering = enforceSnapshotOrdering ? new SnapshotRevisionGate() : null;
_childShutdownTimeout = childShutdownTimeout;
_scriptedAlarmRunner = scriptedAlarmRunner;
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
@@ -2552,8 +2556,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// the cached role unchanged; when this node has a real driver peer (count &gt; 1) that omission is the
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
/// <summary>Gitea #717: the snapshot order gate under Warm; null (no ordering, today's behaviour) otherwise.</summary>
private readonly SnapshotRevisionGate? _snapshotOrdering;
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
{
// Gitea #717 spec §4.2, under OpcUa:Redundancy:Mode=Warm only (Unset/None keep today's unconditional replace):
// a snapshot older than the last accepted one — a late heartbeat of an old singleton incarnation — is dropped,
// so it cannot overwrite the new incarnation's roles. An unrevisioned snapshot (pre-#717 publisher) still feeds
// the gates, as before.
if (_snapshotOrdering?.Offer(msg) == SnapshotRejection.OlderRevision)
{
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
return;
}
_redundancySnapshotReceived = true;
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
if (local is not null)
@@ -14,6 +14,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
@@ -326,7 +327,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null) =>
IOpcUaEndpointState? endpointState = null,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -341,7 +344,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
dbLess,
dataPlaneAuthority,
utcNow,
endpointState)).WithDispatcher(DispatcherId);
endpointState,
redundancy,
standbyProjection)).WithDispatcher(DispatcherId);
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
@@ -385,7 +390,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null) =>
IOpcUaEndpointState? endpointState = null,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -400,7 +407,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
dbLess,
dataPlaneAuthority,
utcNow,
endpointState));
endpointState,
redundancy,
standbyProjection));
/// <summary>Test-only Props exposing the deferred-write bound and the plan-step seams.</summary>
internal static Props PropsForTests(
@@ -461,10 +470,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null)
IOpcUaEndpointState? endpointState = null,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null)
: this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow,
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null,
endpointState)
endpointState, redundancy, standbyProjection)
{
}
@@ -488,8 +499,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Func<DateTime>? utcNow,
int deferredWriteCapacity,
TestHooks? hooks,
IOpcUaEndpointState? endpointState)
IOpcUaEndpointState? endpointState,
WarmRedundancySettings? redundancy = null,
IStandbyProjection? standbyProjection = null)
{
_redundancy = redundancy ?? WarmRedundancySettings.Unset;
_standbyProjection = standbyProjection;
_deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity));
_hooks = hooks;
_sink = sink;
@@ -523,6 +538,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Receive<HealthTick>(_ => OnHealthTick());
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(HandlePeerProbe);
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
// Gitea #717 spec §4.1 / owner answer Q2: under Warm the verdict re-derives on membership and reachability, not
// only on snapshots. Nothing subscribes these under Unset/None.
Receive<ClusterEvent.IMemberEvent>(_ => RecomputeServiceLevel());
Receive<ClusterEvent.IReachabilityEvent>(_ => RecomputeServiceLevel());
Receive<ClusterEvent.CurrentClusterState>(_ => RecomputeServiceLevel());
Receive<WarmRetry>(_ => RecomputeServiceLevel());
}
/// <summary>
@@ -543,6 +564,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
base.PreRestart(reason, message);
}
/// <inheritdoc />
protected override void PostStop()
{
if (_redundancy.IsWarm)
{
try { _cluster.Unsubscribe(Self); }
catch (Exception) { /* the cluster is shutting down with us */ }
}
base.PostStop();
}
/// <inheritdoc />
protected override void PreStart()
{
@@ -551,6 +584,20 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
}
if (_redundancy.IsWarm)
{
OtOpcUaTelemetry.SetWarmRole(0);
try
{
_cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents,
typeof(ClusterEvent.IMemberEvent), typeof(ClusterEvent.IReachabilityEvent));
}
catch (Exception ex)
{
_log.Warning(ex, "OpcUaPublish: Warm redundancy could not subscribe to cluster membership; the verdict re-derives on snapshots only");
}
}
// DB-backed nodes refresh their cached probe status; DB-less nodes still need periodic
// snapshot/probe-verdict age checks even when no new messages arrive. Legacy nodes keep
// their role-only behavior without a timer.
@@ -672,7 +719,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
}
private void HandleRebuild(RebuildAddressSpace msg) => StartRebuild(msg, Sender);
private void HandleRebuild(RebuildAddressSpace msg)
{
if (msg.Artifact is { Length: > 0 } artifact) CheckRedundancyMode(artifact);
StartRebuild(msg, Sender);
}
/// <summary>
/// The entry half of a rebuild, on the actor thread. The raw-sink fallback (no applier) and the
@@ -1465,26 +1516,34 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
}
private void HandleServiceLevelChanged(ServiceLevelChanged msg)
private void HandleServiceLevelChanged(ServiceLevelChanged msg) => TryPublishServiceLevel(msg.ServiceLevel);
/// <summary>The dedup/publish/metric step behind <see cref="ServiceLevelChanged"/>; synchronous so a Warm
/// transition can order it against the projection steps (spec §4.7).</summary>
/// <param name="level">The level to publish.</param>
/// <returns>True when the level is published (or already was).</returns>
private bool TryPublishServiceLevel(byte level)
{
// Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
if (_publishedAtLeastOnce && level == _lastServiceLevel) return true;
try
{
_serviceLevel.Publish(msg.ServiceLevel);
_lastServiceLevel = msg.ServiceLevel;
_serviceLevel.Publish(level);
_lastServiceLevel = level;
_publishedAtLeastOnce = true;
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
_log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel);
new KeyValuePair<string, object?>("level", level));
_log.Debug("OpcUaPublish: ServiceLevel={Level}", level);
return true;
}
catch (Exception ex)
{
// A failed write may be partially applied; even a previous successful value must retry.
_publishedAtLeastOnce = false;
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", msg.ServiceLevel);
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", level);
return false;
}
}
@@ -1494,6 +1553,28 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// wired) or via the legacy role-only seam (back-compat / bootstrap).</summary>
private void HandleRedundancyStateChanged(RedundancyStateChanged msg)
{
if (_redundancy.IsWarm)
{
// Gitea #717 spec §4.2: under Warm a snapshot older than the last accepted one (a late heartbeat from an old
// singleton incarnation) is dropped outright. An unrevisioned one (a pre-#717 publisher mid-upgrade) still
// feeds the health inputs but can never make this node Active. Unset/None keep today's unconditional
// replace below.
switch (_revisionGate.Offer(msg))
{
case SnapshotRejection.OlderRevision:
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
_log.Debug("OpcUaPublish: dropped redundancy snapshot {Revision} (last accepted {Last})",
msg.Revision, _revisionGate.LastAccepted);
return;
case SnapshotRejection.Unrevisioned:
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "unrevisioned"));
break;
default:
_acceptedSnapshot = msg;
break;
}
}
_lastSnapshot = msg;
RecomputeServiceLevel();
}
@@ -1576,6 +1657,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
private void RecomputeServiceLevel()
{
if (_localNode is null || _lastSnapshot is null) return;
if (_redundancy.IsWarm)
{
RecomputeWarm();
return;
}
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
@@ -1635,6 +1721,197 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
}
// ---------------------------------------------------------------- Warm redundancy (Gitea #717)
private sealed class WarmRetry { public static readonly WarmRetry Instance = new(); private WarmRetry() { } }
private readonly WarmRedundancySettings _redundancy;
private readonly IStandbyProjection? _standbyProjection;
private readonly SnapshotRevisionGate _revisionGate = new();
/// <summary>The latest ACCEPTED revisioned snapshot — the only one the Warm verdict reads (spec §4.2).</summary>
private RedundancyStateChanged? _acceptedSnapshot;
/// <summary>The role the data plane is in. A Warm node manager boots Standby, so this starts Standby.</summary>
private WarmRole _warmRole = WarmRole.Standby;
/// <summary>A promotion whose Active ServiceLevel publish failed (spec §4.7): the data plane is Active and the
/// level is retried; the Standby level stays visible meanwhile, never the reverse.</summary>
private bool _promotionLevelPending;
private DateTime? _disagreeSinceUtc;
private bool _disagreementWarned;
/// <summary>How long membership and snapshot may disagree before a Warning (two heartbeats, spec §4.1).</summary>
internal static readonly TimeSpan WarmDisagreementWarning = TimeSpan.FromSeconds(20);
/// <summary>The current Warm role (test observation).</summary>
internal WarmRole CurrentWarmRole => _warmRole;
/// <summary>A promoted node whose Active ServiceLevel publish is being retried (test observation).</summary>
internal bool PromotionLevelPending => _promotionLevelPending;
/// <summary>
/// The Warm ServiceLevel + role path (spec §4.1, §4.3, §4.7): the health facts the non-Warm paths use, the
/// Warm verdict, then a transition in the fixed step order when the verdict changed, else just the band.
/// </summary>
private void RecomputeWarm()
{
var self = _localNode!.Value;
var entry = _lastSnapshot!.Nodes.FirstOrDefault(n => n.NodeId == self);
var now = _utcNow();
var memberState = SafeSelfStatus();
var listening = EndpointListening();
bool dbReachable;
bool stale;
if (_dbLess || _dbHealthProbe is null || _lastDbHealth is null)
{
dbReachable = true;
stale = entry is null || (now - entry.AsOfUtc) > _staleWindow;
}
else
{
dbReachable = _lastDbHealth.Reachable;
stale = !_lastDbHealth.Reachable
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|| entry is null || (now - entry.AsOfUtc) > _staleWindow;
}
var verdict = EvaluateWarmVerdict(self, memberState, listening);
TrackDisagreement(verdict, now);
var detached = entry is null || entry.Role == RedundancyRole.Detached;
var inputs = new NodeHealthInputs(memberState, dbReachable, OpcUaProbeOk(), stale, IsDriverPrimary: false, listening);
var target = detached ? WarmRole.Standby : verdict.Role;
var level = detached ? (byte)0 : ServiceLevelCalculator.ComputeWarm(inputs, target == WarmRole.Active);
ApplyWarmTransition(target, level, verdict.Reason);
}
private WarmVerdict EvaluateWarmVerdict(NodeId self, MemberStatus memberState, bool listening)
{
NodeId? oldest = null, oldestReachable = null;
var primaryUnreachable = false;
var snapshotPrimary = _acceptedSnapshot?.Nodes.FirstOrDefault(n => n.Role == RedundancyRole.Primary)?.NodeId;
try
{
var state = _cluster.State;
var unreachable = state.Unreachable.ToHashSet();
oldest = WarmRoleAuthority.OldestUpMember(state.Members, "driver");
oldestReachable = WarmRoleAuthority.OldestUpMember(state.Members, "driver", unreachable);
primaryUnreachable = snapshotPrimary is { } sp && unreachable.Any(m => WarmRoleAuthority.ToNodeId(m) == sp);
}
catch (Exception ex)
{
_log.Debug(ex, "OpcUaPublish: cluster state unavailable for the Warm verdict; treating membership as unknown");
}
return WarmRoleAuthority.Evaluate(new WarmRoleInputs(
self, memberState == MemberStatus.Up, listening, oldest, oldestReachable,
HasAcceptedSnapshot: _acceptedSnapshot is not null, snapshotPrimary, primaryUnreachable,
_redundancy.PromoteWhilePrimaryUnreachable));
}
private void TrackDisagreement(WarmVerdict verdict, DateTime now)
{
if (!verdict.Disagreement)
{
_disagreeSinceUtc = null;
_disagreementWarned = false;
return;
}
_disagreeSinceUtc ??= now;
if (_disagreementWarned || now - _disagreeSinceUtc.Value <= WarmDisagreementWarning) return;
_disagreementWarned = true;
_log.Warning(
"OpcUaPublish: Warm redundancy — cluster membership and the redundancy snapshot (revision {Revision}) have disagreed about this node being Primary for more than {Seconds}s; the node stays {Role}",
_revisionGate.LastAccepted, WarmDisagreementWarning.TotalSeconds, _warmRole);
}
/// <summary>
/// Moves the data plane to <paramref name="target"/> in the spec §4.7 order and publishes
/// <paramref name="level"/>. Invariant: an Active-band ServiceLevel is never visible while the projection is
/// closed — a promotion publishes the level LAST and, if that publish fails, retries it; a demotion publishes
/// the Standby level FIRST and degrades the data whether or not that publish succeeded (failing safe).
/// </summary>
private void ApplyWarmTransition(WarmRole target, byte level, string reason)
{
if (target == _warmRole)
{
if (!TryPublishServiceLevel(level) && target == WarmRole.Active) ScheduleWarmRetry();
else if (target == WarmRole.Active) _promotionLevelPending = false;
return;
}
var projection = _standbyProjection;
if (target == WarmRole.Active)
{
projection?.SetDataProjection(false); // 1. open the read projection
projection?.QueueCurrentToLiveItems(false); // 2. current Good value to every live data item
projection?.SetStandbyDelivery(false); // 3. first deliveries decided Active
projection?.SetAlarmSuppression(false); // 4. new alarm events delivered
projection?.RefreshAllEventItems(); // 5. promotion refresh on every live event item
_warmRole = WarmRole.Active;
if (TryPublishServiceLevel(level)) // 6. Active ServiceLevel, last
{
_promotionLevelPending = false;
}
else
{
_promotionLevelPending = true;
ScheduleWarmRetry();
}
}
else
{
TryPublishServiceLevel(level); // 1. Standby ServiceLevel, first (continue on failure)
projection?.SetStandbyDelivery(true); // 2. the D4 boundary
projection?.SetAlarmSuppression(true); // 3. suppress new alarm events
projection?.SetDataProjection(true); // 4. close the read projection
projection?.QueueCurrentToLiveItems(true); // 5. BadOutOfService to every live data item
_warmRole = WarmRole.Standby;
_promotionLevelPending = false;
}
OtOpcUaTelemetry.SetWarmRole(_warmRole == WarmRole.Active ? 1 : 0);
OtOpcUaTelemetry.WarmTransition.Add(1,
new KeyValuePair<string, object?>("direction", target == WarmRole.Active ? "promote" : "demote"),
new KeyValuePair<string, object?>("reason", reason));
_log.Info(
"OpcUaPublish: Warm redundancy {Direction} to {Role} (reason={Reason}, revision={Revision}, ServiceLevel={Level})",
target == WarmRole.Active ? "promotion" : "demotion", _warmRole, reason, _revisionGate.LastAccepted, level);
}
/// <summary>
/// Gitea #717 spec §3 deploy-time cross-check: under <c>None</c> or <c>Warm</c>, an artifact whose cluster
/// <c>RedundancyMode</c> disagrees with the local mode (<c>Hot</c> included, owner answer Q5) logs an Error and
/// counts <c>otopcua.redundancy.mode_mismatch</c>. The node keeps its local mode — the only value available
/// before SDK start. <c>Unset</c> has opted out and is not checked.
/// </summary>
/// <param name="artifact">The applied artifact bytes.</param>
private void CheckRedundancyMode(byte[] artifact)
{
if (_redundancy.Mode == OpcUaRedundancyMode.Unset || _localNode is null) return;
string? clusterMode;
try
{
clusterMode = DeploymentArtifact.ReadClusterRedundancyMode(artifact, _localNode.Value.Value);
}
catch (Exception ex)
{
_log.Debug(ex, "OpcUaPublish: could not read the artifact's cluster RedundancyMode");
return;
}
if (clusterMode is null) return;
if (string.Equals(clusterMode, _redundancy.Mode.ToString(), StringComparison.OrdinalIgnoreCase)) return;
OtOpcUaTelemetry.RedundancyModeMismatch.Add(1,
new KeyValuePair<string, object?>("local", _redundancy.Mode.ToString()),
new KeyValuePair<string, object?>("cluster", clusterMode));
_log.Error(
"OpcUaPublish: the deployed ServerCluster.RedundancyMode for this node's cluster is {ClusterMode} but OpcUa:Redundancy:Mode is {LocalMode}; the node keeps running as {LocalMode} (local configuration is the authority — fix whichever is wrong)",
clusterMode, _redundancy.Mode, _redundancy.Mode);
}
private void ScheduleWarmRetry() => Timers.StartSingleTimer("warm-retry", WarmRetry.Instance, TimeSpan.FromSeconds(1));
/// <summary>The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200,
/// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
@@ -0,0 +1,109 @@
using Akka.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
/// <summary>The Warm role (Gitea #717 spec §4.1).</summary>
public enum WarmRole
{
/// <summary>Degraded standby: projected data, suppressed new alarm events, Degraded-band ServiceLevel.</summary>
Standby,
/// <summary>Serves data and alarms; Healthy-band ServiceLevel.</summary>
Active,
}
/// <summary>The facts the Warm verdict is a pure function of.</summary>
/// <param name="Self">This node's canonical <c>host:port</c> id.</param>
/// <param name="SelfUp">This node's own member status is Up.</param>
/// <param name="EndpointListening">This node's OPC UA endpoint did not fail to start (#762).</param>
/// <param name="OldestUpDriver">The oldest Up <c>driver</c> member in this node's OWN cluster state (the singleton
/// placement rule), or null when there is none.</param>
/// <param name="OldestReachableUpDriver">The same, ignoring members this node sees as unreachable.</param>
/// <param name="HasAcceptedSnapshot">A revisioned snapshot has been accepted (spec §4.2). Unrevisioned snapshots
/// never count.</param>
/// <param name="SnapshotPrimary">The Primary named by the latest accepted revisioned snapshot, or null.</param>
/// <param name="SnapshotPrimaryUnreachable">This node sees <paramref name="SnapshotPrimary"/> as unreachable.</param>
/// <param name="PromoteWhilePrimaryUnreachable">The owner-answer-Q2 option (default on).</param>
public readonly record struct WarmRoleInputs(
NodeId Self,
bool SelfUp,
bool EndpointListening,
NodeId? OldestUpDriver,
NodeId? OldestReachableUpDriver,
bool HasAcceptedSnapshot,
NodeId? SnapshotPrimary,
bool SnapshotPrimaryUnreachable,
bool PromoteWhilePrimaryUnreachable);
/// <summary>The verdict and why.</summary>
/// <param name="Role">Active or Standby.</param>
/// <param name="Reason">A short reason for the transition metric/log: <c>boot</c>, <c>endpoint</c>,
/// <c>membership</c>, <c>snapshot</c>, <c>unreachable-primary</c> or <c>agreed</c>.</param>
/// <param name="Disagreement">Membership (2) and snapshot (3) disagree about whether this node is Primary.</param>
public readonly record struct WarmVerdict(WarmRole Role, string Reason, bool Disagreement);
/// <summary>
/// The Warm role verdict (Gitea #717 spec §4.1), a pure function so every row is unit-testable. Active iff the
/// node is Up and listening, its own cluster state names it the oldest Up driver member, and the latest accepted
/// revisioned snapshot names it Primary. Anything else — no snapshot yet, an unrevisioned snapshot, a
/// disagreement — is Standby. With <see cref="WarmRoleInputs.PromoteWhilePrimaryUnreachable"/> the survivor of a
/// crashed Primary goes Active once that Primary is unreachable, before <c>auto-down</c> removes it.
/// </summary>
/// <remarks>
/// This is a separate output from <c>PrimaryGatePolicy</c>, whose lone-node allow is unchanged for writes and
/// acks (D7).
/// </remarks>
public static class WarmRoleAuthority
{
/// <summary>Evaluates the verdict.</summary>
/// <param name="i">The inputs.</param>
/// <returns>The verdict.</returns>
public static WarmVerdict Evaluate(in WarmRoleInputs i)
{
if (!i.SelfUp) return new(WarmRole.Standby, "membership", false);
if (!i.EndpointListening) return new(WarmRole.Standby, "endpoint", false);
if (!i.HasAcceptedSnapshot) return new(WarmRole.Standby, "boot", false);
var membershipSaysSelf = i.OldestUpDriver == i.Self;
var snapshotSaysSelf = i.SnapshotPrimary == i.Self;
var disagreement = membershipSaysSelf != snapshotSaysSelf;
if (membershipSaysSelf && snapshotSaysSelf) return new(WarmRole.Active, "agreed", false);
// Owner answer Q2: a SIGKILLed Primary stays an Up-but-unreachable member until auto-down removes it, and the
// singleton usually died with it. Evaluate (2) and (3) with unreachable members excluded.
if (i.PromoteWhilePrimaryUnreachable
&& i.OldestReachableUpDriver == i.Self
&& (snapshotSaysSelf || (i.SnapshotPrimary is not null && i.SnapshotPrimaryUnreachable)))
{
return new(WarmRole.Active, "unreachable-primary", disagreement);
}
return new(WarmRole.Standby, membershipSaysSelf ? "snapshot" : "membership", disagreement);
}
/// <summary>The oldest Up member carrying <paramref name="role"/> (the singleton placement rule), optionally
/// ignoring <paramref name="unreachable"/> members.</summary>
/// <param name="members">The current members.</param>
/// <param name="role">The role.</param>
/// <param name="unreachable">Members to ignore, or null.</param>
/// <returns>Its canonical node id, or null.</returns>
public static NodeId? OldestUpMember(IEnumerable<Member> members, string role, IReadOnlySet<Member>? unreachable = null)
{
Member? oldest = null;
foreach (var m in members)
{
if (m.Status != MemberStatus.Up || !m.HasRole(role)) continue;
if (unreachable is not null && unreachable.Contains(m)) continue;
if (oldest is null || m.IsOlderThan(oldest)) oldest = m;
}
return oldest is null ? null : ToNodeId(oldest);
}
/// <summary>The canonical <c>host:port</c> id of a member (the RedundancyStateActor format).</summary>
/// <param name="m">The member.</param>
/// <returns>The node id.</returns>
public static NodeId ToNodeId(Member m) => NodeId.Parse($"{m.Address.Host ?? string.Empty}:{m.Address.Port ?? 0}");
}
@@ -566,6 +566,9 @@ public static class ServiceCollectionExtensions
// Optional: a harness that builds this graph without the hosted service registers none, and both
// consumers then behave as if the endpoint were listening (the pre-#762 behaviour).
var endpointState = resolver.GetService<IOpcUaEndpointState>();
// Gitea #717: OpcUa:Redundancy (registered by the Host from the validated OpcUa options). Absent in harnesses ⇒
// Unset ⇒ today's behaviour. The standby projection is late-bound like the ServiceLevel publisher.
var redundancy = resolver.GetService<WarmRedundancySettings>() ?? WarmRedundancySettings.Unset;
var publishActor = system.ActorOf(
OpcUaPublishActor.Props(
sink: addressSpaceSink,
@@ -576,7 +579,9 @@ public static class ServiceCollectionExtensions
dbHealthProbe: dbHealth,
dbLess: dbFactory is null,
dataPlaneAuthority: dataPlaneAuthority,
endpointState: endpointState),
endpointState: endpointState,
redundancy: redundancy,
standbyProjection: redundancy.IsWarm ? resolver.GetService<DeferredStandbyProjection>() : null),
OpcUaPublishActorName);
registry.Register<OpcUaPublishActorKey>(publishActor);
@@ -670,7 +675,9 @@ public static class ServiceCollectionExtensions
// lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
// the InProcess default, when the seam is not listed, and in test graphs that register no provider.
scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm),
// Gitea #717 spec §4.2: drop older redundancy snapshots under Warm only.
enforceSnapshotOrdering: redundancy.IsWarm),
DriverHostActorName);
registry.Register<DriverHostActorKey>(driverHost);