feat(717): Warm non-transparent redundancy, default-off (T1-T10)
OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start). - T1 options + validator + effective-URI check + boot banner; Warm needs the driver role. - T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None and the SDK's orphan i=11314 deleted; Unset untouched. - T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active. - T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable. - T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec 4.7 order (Active level last and retried; Standby level first, data degraded regardless). - T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState. - T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's cached copy is never mutated. - T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters. - T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion refresh via ISubscription.ConditionRefresh2Async on every live event item. - T10 deploy-time RedundancyMode cross-check + six instruments. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -37,6 +37,11 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
private readonly Action<object>? _broadcastOverride;
|
||||
private readonly TimeSpan _heartbeatInterval;
|
||||
private bool _dirty;
|
||||
/// <summary>Gitea #717 spec §4.2: <c>Environment.TickCount64</c> at <see cref="PreStart"/>, ordering a singleton
|
||||
/// restarted in the same process.</summary>
|
||||
private long _incarnationTick;
|
||||
/// <summary>Gitea #717 spec §4.2: incremented on every publish, heartbeats included.</summary>
|
||||
private long _sequence;
|
||||
|
||||
/// <summary>Gets the timer scheduler for this actor.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
@@ -78,6 +83,7 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
_incarnationTick = NextIncarnationTick();
|
||||
_cluster.Subscribe(
|
||||
Self,
|
||||
ClusterEvent.InitialStateAsEvents,
|
||||
@@ -107,12 +113,63 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
_dirty = false;
|
||||
|
||||
var snapshot = BuildSnapshot();
|
||||
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId());
|
||||
var msg = new RedundancyStateChanged(snapshot, CorrelationId.NewId(), NextRevision());
|
||||
if (_broadcastOverride is not null) _broadcastOverride(msg);
|
||||
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(Topic, msg));
|
||||
_log.Debug("Published RedundancyStateChanged with {Count} nodes", snapshot.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ordered revision of the next publish (Gitea #717 spec §4.2): the up-number of the member hosting this
|
||||
/// singleton, this incarnation's start tick, and a per-publish sequence. Subscribers drop anything not newer,
|
||||
/// so a late heartbeat from an old incarnation can no longer overwrite the new incarnation's snapshot.
|
||||
/// </summary>
|
||||
private SnapshotRevision NextRevision()
|
||||
{
|
||||
long upNumber;
|
||||
try
|
||||
{
|
||||
upNumber = ReadUpNumber(_cluster.SelfMember)
|
||||
?? throw new InvalidOperationException("Akka.Cluster Member.UpNumber is not readable on this Akka version");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Not expected on a running singleton; an unordered 0 can never outrank a real publisher.
|
||||
_log.Debug(ex, "RedundancyState: SelfMember up-number unavailable; publishing revision up-number 0");
|
||||
upNumber = 0;
|
||||
}
|
||||
|
||||
return new SnapshotRevision(upNumber, _incarnationTick, ++_sequence);
|
||||
}
|
||||
|
||||
private static long _lastIncarnationTick;
|
||||
|
||||
/// <summary><c>Environment.TickCount64</c>, made strictly increasing within the process so a singleton restarted
|
||||
/// inside one tick still outranks its predecessor (whose sequence may be ahead of the new one's).</summary>
|
||||
/// <returns>The incarnation tick.</returns>
|
||||
public static long NextIncarnationTick()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var last = Interlocked.Read(ref _lastIncarnationTick);
|
||||
var next = Math.Max(Environment.TickCount64, last + 1);
|
||||
if (Interlocked.CompareExchange(ref _lastIncarnationTick, next, last) == last) return next;
|
||||
}
|
||||
}
|
||||
|
||||
private static readonly System.Reflection.PropertyInfo? UpNumberProperty = typeof(Member).GetProperty(
|
||||
"UpNumber", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic);
|
||||
|
||||
/// <summary>
|
||||
/// A member's up-number — the order in which members became Up, the basis of <c>Member.IsOlderThan</c> and of
|
||||
/// singleton placement. Akka.NET 1.5 keeps the property <c>internal</c>, so it is read by reflection; the
|
||||
/// <c>SnapshotRevisionTests</c> pin that it resolves, so an Akka upgrade that renames it fails loudly.
|
||||
/// </summary>
|
||||
/// <param name="member">The member.</param>
|
||||
/// <returns>The up-number, or null if the property is unavailable.</returns>
|
||||
public static int? ReadUpNumber(Member member) =>
|
||||
UpNumberProperty?.GetValue(member) is int value ? value : null;
|
||||
|
||||
/// <summary>The cluster role that marks a node as carrying the driver runtime.</summary>
|
||||
public const string DriverRole = RoleParser.Driver;
|
||||
|
||||
|
||||
+5
@@ -60,5 +60,10 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase<
|
||||
|| lockout.MaxFailedAttemptsPerClient >= lockout.MaxFailedAttempts,
|
||||
"OpcUa:CredentialLockout:MaxFailedAttemptsPerClient must be 0 (= 2 x MaxFailedAttempts) or at least "
|
||||
+ "OpcUa:CredentialLockout:MaxFailedAttempts");
|
||||
|
||||
// Gitea #717 spec §3: Hot/unknown modes fail start; Warm needs distinct peers that do not include this server's
|
||||
// own ApplicationUri; None needs no peers. The effective URI is re-checked after an ApplicationConfigPath load.
|
||||
foreach (var failure in (o.Redundancy ?? new OpcUaRedundancyOptions()).Check(o.ApplicationUri, o.PeerApplicationUris))
|
||||
builder.RequireThat(false, failure);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// The one Gitea #717 startup rule the options validator cannot see: <c>OpcUa:Redundancy:Mode=Warm</c> needs the
|
||||
/// <c>driver</c> role (the validator is registered only on driver nodes, so on any other node it would never run).
|
||||
/// </summary>
|
||||
public static class OpcUaRedundancyStartupGuard
|
||||
{
|
||||
/// <summary>Fails host start when a node without the driver role is configured Warm.</summary>
|
||||
/// <param name="configuredMode">The raw <c>OpcUa:Redundancy:Mode</c> value.</param>
|
||||
/// <param name="hasDriver">Whether this node carries the driver role.</param>
|
||||
/// <exception cref="InvalidOperationException">Warm without the driver role.</exception>
|
||||
public static void RequireDriverRoleForWarm(string? configuredMode, bool hasDriver)
|
||||
{
|
||||
if (hasDriver) return;
|
||||
var options = new OpcUaRedundancyOptions { Mode = configuredMode ?? string.Empty };
|
||||
if (options.TryParseMode(out var mode) && mode == OpcUaRedundancyMode.Warm)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"OpcUa:Redundancy:Mode is Warm on a node without the driver role; Warm redundancy needs the driver role "
|
||||
+ "(the OPC UA endpoint is served by driver nodes only). Remove the setting or add the driver role.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
private readonly OpcUaApplicationHostOptions _options;
|
||||
private readonly DeferredAddressSpaceSink _deferredSink;
|
||||
private readonly DeferredServiceLevelPublisher _deferredServiceLevel;
|
||||
private readonly DeferredStandbyProjection? _deferredStandby;
|
||||
private readonly IOpcUaUserAuthenticator _userAuthenticator;
|
||||
private readonly IHistorianDataSource _historianDataSource;
|
||||
private readonly IDataPlaneAuthoritySource _dataPlaneAuthority;
|
||||
@@ -127,6 +128,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
/// <param name="writeBounds">The node's configured write bounds (<c>DriverWrites:Timeout</c>); the inbound
|
||||
/// write gateway's Ask is derived from them (lmxopcua#777). Optional: null ⇒
|
||||
/// <see cref="DriverWriteBounds.Default"/>.</param>
|
||||
/// <param name="deferredStandby">The late-bound Warm standby projection (Gitea #717) <c>OpcUaPublishActor</c> drives;
|
||||
/// bound here once the SDK has started. Optional: null (and a non-Warm server) binds nothing.</param>
|
||||
public OtOpcUaServerHostedService(
|
||||
IOptions<OpcUaApplicationHostOptions> options,
|
||||
DeferredAddressSpaceSink deferredSink,
|
||||
@@ -147,8 +150,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
OpcUaEndpointState? endpointState = null,
|
||||
EffectiveRoles? roles = null,
|
||||
NativeAlarmAckPreflightView? ackPreflightView = null,
|
||||
DriverWriteBounds? writeBounds = null)
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
DeferredStandbyProjection? deferredStandby = null)
|
||||
{
|
||||
_deferredStandby = deferredStandby;
|
||||
ArgumentNullException.ThrowIfNull(aclOptions);
|
||||
ArgumentNullException.ThrowIfNull(time);
|
||||
|
||||
@@ -402,6 +407,10 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
serverInternal,
|
||||
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
|
||||
}
|
||||
|
||||
// Gitea #717: the Warm standby projection. Bound right after the ServiceLevel so a transition the actor
|
||||
// already decided replays onto the node manager; null (every non-Warm server) leaves the adapter unbound.
|
||||
if (_server.CreateStandbyProjection() is { } standbyProjection) _deferredStandby?.SetInner(standbyProjection);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -568,6 +577,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// half-disposed NodeManager.
|
||||
_deferredSink.SetSink(null);
|
||||
_deferredServiceLevel.SetInner(null);
|
||||
_deferredStandby?.SetInner(null);
|
||||
// Restore the Null write gateway so a late client write doesn't Ask a stopping DriverHostActor.
|
||||
_server?.SetNodeWriteGateway(null);
|
||||
// Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor.
|
||||
|
||||
@@ -206,6 +206,10 @@ ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.OpcUaUserCertificatePaths.Configure(
|
||||
// lmxopcua#765: refresh/staleness of the cached API-key → Galaxy ack identity set (driver nodes consume it).
|
||||
builder.Services.AddGalaxyAckIdentityCache(builder.Configuration);
|
||||
|
||||
// Gitea #717 spec §3: Warm redundancy is a property of the OPC UA endpoint, which only a driver node serves. A Warm
|
||||
// mode on a node without the driver role is a configuration error, not something to ignore silently.
|
||||
OpcUaRedundancyStartupGuard.RequireDriverRoleForWarm(builder.Configuration["OpcUa:Redundancy:Mode"], hasDriver);
|
||||
|
||||
if (hasDriver)
|
||||
{
|
||||
builder.Services.AddOtOpcUaRuntime();
|
||||
@@ -501,6 +505,11 @@ if (hasDriver)
|
||||
builder.Services.AddValidatedOptions<OpcUaApplicationHostOptions, OpcUaApplicationHostOptionsValidator>(
|
||||
builder.Configuration, "OpcUa");
|
||||
|
||||
// Gitea #717: the runtime half of OpcUa:Redundancy for the redundancy actors, and the late-bound standby
|
||||
// projection the OPC UA hosted service binds once the SDK has started (same pattern as the ServiceLevel publisher).
|
||||
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<OpcUaApplicationHostOptions>>().Value.Redundancy.ToSettings());
|
||||
builder.Services.AddSingleton<DeferredStandbyProjection>();
|
||||
|
||||
// Lazy ActorSystem accessor so OtOpcUaServerHostedService can resolve the DistributedPubSub
|
||||
// mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
|
||||
// DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
|
||||
|
||||
@@ -6,8 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// Preserves the application's established equipment-write conversions across SDK 2.0's
|
||||
/// new pre-handler type validation. The owning callback remains the authorization and shape gate.
|
||||
/// </summary>
|
||||
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock) : BaseDataVariableState(parent)
|
||||
internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock, WarmStandbyState? standby = null)
|
||||
: BaseDataVariableState(parent)
|
||||
{
|
||||
private readonly WarmStandbyState _standby = standby ?? WarmStandbyState.Off;
|
||||
|
||||
// Every field below is accessed under ownerLock. Tokens identify client attempts; the mutation
|
||||
// revision detects a driver publication that re-enters the callback before its optimistic commit.
|
||||
private long _nextWriteToken;
|
||||
@@ -82,6 +85,16 @@ internal sealed class EquipmentVariableState(NodeState? parent, object ownerLock
|
||||
QualifiedName dataEncoding, ref Variant value, ref DateTimeUtc sourceTimestamp)
|
||||
{
|
||||
// Subscription samplers read the node directly, bypassing the manager's service lock.
|
||||
lock (ownerLock) return base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
|
||||
ServiceResult result;
|
||||
lock (ownerLock) result = base.ReadValueAttribute(context, indexRange, dataEncoding, ref value, ref sourceTimestamp);
|
||||
|
||||
// Gitea #717 spec §4.4 (T0b): on a Warm Standby every value path — Read (IndexRange and DataEncoding included),
|
||||
// the initial sample, the change fan-out, ResendData, AggregateFilter input — answers BadOutOfService/null. The
|
||||
// stored Value is untouched and keeps updating from acquisition. The base read runs first, so an IndexRange or
|
||||
// encoding error is still reported as such. Off (Unset/None/Active): returned unchanged.
|
||||
if (ServiceResult.IsBad(result) || !_standby.DataProjected) return result;
|
||||
value = Variant.Null;
|
||||
sourceTimestamp = DateTimeUtc.MinValue;
|
||||
return StatusCodes.BadOutOfService;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using Opc.Ua;
|
||||
using Opc.Ua.Configuration;
|
||||
using Opc.Ua.Identity;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
|
||||
@@ -174,6 +175,83 @@ public sealed class OpcUaApplicationHostOptions
|
||||
/// Replaces the SDK's own lockout, which also refused Anonymous; see <see cref="FailedCredentialLockout"/>.
|
||||
/// </summary>
|
||||
public OpcUaCredentialLockoutOptions CredentialLockout { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Warm non-transparent redundancy (<c>OpcUa:Redundancy:*</c>, Gitea #717). The default <c>Mode=Unset</c> is
|
||||
/// byte-for-byte today's server. Startup-static.
|
||||
/// </summary>
|
||||
public OpcUaRedundancyOptions Redundancy { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>The <c>OpcUa:Redundancy</c> section (Gitea #717, spec §6).</summary>
|
||||
public sealed class OpcUaRedundancyOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>Unset</c> (default) / <c>None</c> / <c>Warm</c>, case-insensitive. <c>Hot</c> is not advertisable
|
||||
/// (spec §3) and, like any other value, fails host start. Bound as a string so a bad value is a readable
|
||||
/// validation failure rather than a binder exception.
|
||||
/// </summary>
|
||||
public string Mode { get; set; } = nameof(OpcUaRedundancyMode.Unset);
|
||||
|
||||
/// <summary>
|
||||
/// Warm only: let the survivor go Active while the Primary is unreachable but not yet downed (spec §4.1, owner
|
||||
/// answer Q2). Default <c>true</c>.
|
||||
/// </summary>
|
||||
public bool PromoteWhilePrimaryUnreachable { get; set; } = true;
|
||||
|
||||
/// <summary>Parses <see cref="Mode"/>.</summary>
|
||||
/// <param name="mode">The parsed mode.</param>
|
||||
/// <returns><c>false</c> for <c>Hot</c>, numerics or any unknown value.</returns>
|
||||
public bool TryParseMode(out OpcUaRedundancyMode mode)
|
||||
{
|
||||
var text = (Mode ?? string.Empty).Trim();
|
||||
if (text.Length == 0) text = nameof(OpcUaRedundancyMode.Unset);
|
||||
if (!text.All(char.IsLetter)) { mode = default; return false; }
|
||||
return Enum.TryParse(text, ignoreCase: true, out mode) && Enum.IsDefined(mode);
|
||||
}
|
||||
|
||||
/// <summary>The parsed mode; throws on an invalid one (the validator rejects it first).</summary>
|
||||
public OpcUaRedundancyMode ParsedMode => TryParseMode(out var m)
|
||||
? m
|
||||
: throw new InvalidOperationException(
|
||||
$"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable).");
|
||||
|
||||
/// <summary>The runtime settings the redundancy actors read.</summary>
|
||||
/// <returns>The settings.</returns>
|
||||
public WarmRedundancySettings ToSettings() => new(ParsedMode, PromoteWhilePrimaryUnreachable);
|
||||
|
||||
/// <summary>
|
||||
/// The rules of spec §3 that need only configuration: <c>Warm</c> needs at least one peer, all URIs distinct
|
||||
/// and the self URI not listed as a peer; <c>None</c> needs no peers. Returns the failure messages (empty when
|
||||
/// valid). The effective self URI is checked again after an <c>ApplicationConfigPath</c> load.
|
||||
/// </summary>
|
||||
/// <param name="selfUri">The self URI to check against.</param>
|
||||
/// <param name="peers">The configured peer URIs.</param>
|
||||
/// <returns>The failures.</returns>
|
||||
public IReadOnlyList<string> Check(string? selfUri, IEnumerable<string>? peers)
|
||||
{
|
||||
var failures = new List<string>();
|
||||
if (!TryParseMode(out var mode))
|
||||
{
|
||||
failures.Add($"OpcUa:Redundancy:Mode '{Mode}' is not supported; use Unset, None or Warm (Hot is not advertisable: continuous acquisition alone does not meet Part 4 Hot)");
|
||||
return failures;
|
||||
}
|
||||
|
||||
var list = (peers ?? Array.Empty<string>()).Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToList();
|
||||
if (mode == OpcUaRedundancyMode.None && list.Count > 0)
|
||||
failures.Add("OpcUa:PeerApplicationUris must be empty when OpcUa:Redundancy:Mode is None (a standalone server has no peers)");
|
||||
if (mode == OpcUaRedundancyMode.Warm)
|
||||
{
|
||||
if (list.Count == 0)
|
||||
failures.Add("OpcUa:Redundancy:Mode Warm requires at least one OpcUa:PeerApplicationUris entry");
|
||||
if (list.Distinct(StringComparer.Ordinal).Count() != list.Count)
|
||||
failures.Add("OpcUa:PeerApplicationUris must not contain duplicates under OpcUa:Redundancy:Mode Warm");
|
||||
if (!string.IsNullOrWhiteSpace(selfUri) && list.Contains(selfUri.Trim(), StringComparer.Ordinal))
|
||||
failures.Add($"OpcUa:PeerApplicationUris must not list this server's own ApplicationUri '{selfUri}' under OpcUa:Redundancy:Mode Warm (each pair node needs a unique ApplicationUri)");
|
||||
}
|
||||
|
||||
return failures;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -289,7 +367,8 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
|
||||
|
||||
try
|
||||
{
|
||||
_ = await BuildConfigurationAsync(cancellationToken);
|
||||
var configuration = await BuildConfigurationAsync(cancellationToken);
|
||||
ConfigureRedundancy(sdkServer, configuration.ApplicationUri ?? _options.ApplicationUri);
|
||||
await EnsureApplicationCertificateAsync(cancellationToken).ConfigureAwait(false);
|
||||
AttachUserAuthenticator(sdkServer);
|
||||
await _application.StartAsync(server, cancellationToken).ConfigureAwait(false);
|
||||
@@ -305,6 +384,31 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
|
||||
_options.PublicHostname, _options.OpcUaPort);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 (spec §3): validate the redundancy mode against the EFFECTIVE self URI — the loaded
|
||||
/// <c>ApplicationConfigPath</c> XML wins over <c>OpcUa:ApplicationUri</c> (c7787) — log the boot banner and hand
|
||||
/// the startup-static metadata to the SDK server, which installs it in <c>OnNodeManagerStartedAsync</c>.
|
||||
/// <c>Unset</c> hands nothing over, so the SDK's graph is untouched.
|
||||
/// </summary>
|
||||
/// <param name="server">The SDK server about to start.</param>
|
||||
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
|
||||
/// <exception cref="InvalidOperationException">The redundancy configuration is invalid for the effective URI.</exception>
|
||||
private void ConfigureRedundancy(OtOpcUaSdkServer server, string effectiveSelfUri)
|
||||
{
|
||||
var redundancy = _options.Redundancy ?? new OpcUaRedundancyOptions();
|
||||
var failures = redundancy.Check(effectiveSelfUri, _options.PeerApplicationUris);
|
||||
if (failures.Count > 0)
|
||||
throw new InvalidOperationException("OPC UA redundancy configuration is invalid: " + string.Join("; ", failures));
|
||||
|
||||
var mode = redundancy.ParsedMode;
|
||||
var peers = (_options.PeerApplicationUris ?? new List<string>())
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).ToArray();
|
||||
_logger.LogInformation(
|
||||
"OPC UA redundancy: Mode={Mode}, effective ApplicationUri={SelfUri}, peers=[{Peers}], PromoteWhilePrimaryUnreachable={Promote}",
|
||||
mode, effectiveSelfUri, string.Join(", ", peers), redundancy.PromoteWhilePrimaryUnreachable);
|
||||
server.ConfigureRedundancy(mode, effectiveSelfUri, peers);
|
||||
}
|
||||
|
||||
private void AttachUserAuthenticator(OtOpcUaSdkServer server)
|
||||
{
|
||||
foreach (var type in new[] { UserTokenType.Anonymous, UserTokenType.UserName })
|
||||
|
||||
@@ -161,6 +161,34 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
private FolderState? _root;
|
||||
private readonly ILogger<OtOpcUaNodeManager> _logger;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
|
||||
/// <param name="server">The OPC UA server instance.</param>
|
||||
/// <param name="configuration">The application configuration.</param>
|
||||
/// <param name="logger">Logger for the manager's swallowed-fault and skipped-wiring diagnostics;
|
||||
/// <c>null</c> (tests) falls back to <see cref="NullLogger{T}.Instance"/>.</param>
|
||||
/// <param name="standby">The Warm standby flags (Gitea #717); null (every non-Warm server) is
|
||||
/// <see cref="WarmStandbyState.Off"/>, which leaves every data path exactly as it was.</param>
|
||||
internal OtOpcUaNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration, ILogger<OtOpcUaNodeManager>? logger,
|
||||
WarmStandbyState? standby)
|
||||
: this(server, configuration, logger)
|
||||
{
|
||||
_standby = standby ?? WarmStandbyState.Off;
|
||||
if (_standby.WarmMode)
|
||||
{
|
||||
// Spec §4.6 item 1: ordinary-tag Value items are built by OrdinaryTagMonitoredItem, which marks each
|
||||
// notification. Warm only — Unset/None keep the SDK's own manager.
|
||||
m_monitoredItemManager.Dispose();
|
||||
m_monitoredItemManager = new OrdinaryTagItemManager(this.ToAsyncNodeManager(), server);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The Warm standby flags (Gitea #717). <see cref="WarmStandbyState.Off"/> unless the server is Warm.</summary>
|
||||
private readonly WarmStandbyState _standby = WarmStandbyState.Off;
|
||||
|
||||
/// <summary>Test observation: the Warm standby flags.</summary>
|
||||
internal WarmStandbyState Standby => _standby;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OtOpcUaNodeManager"/> class with the OPC UA server and configuration.</summary>
|
||||
/// <param name="server">The OPC UA server instance.</param>
|
||||
/// <param name="configuration">The application configuration.</param>
|
||||
@@ -1434,6 +1462,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
// the InstanceStateSnapshot captures the just-projected values.
|
||||
alarm.ClearChangeMasks(SystemContext, includeChildren: true);
|
||||
|
||||
// Gitea #717 spec §4.5: on a Warm Standby only the STAMP half above runs — the condition node is current and a
|
||||
// later refresh carries a valid, ackable EventId — and the DELIVER half below does not. Suppressing here, before
|
||||
// the snapshot and every SDK field projection, covers the Server-object, folder and direct paths, ACL on or
|
||||
// off, by construction; overflow, filtered retain and the #838 deferral queue never see the event. The
|
||||
// decision is made at report time: an event reported while Active is delivered even if that ends after
|
||||
// demotion.
|
||||
if (_standby.AlarmsSuppressed)
|
||||
{
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbySuppressedAlarmEvents.Add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// InstanceStateSnapshot is the IFilterTarget — a frozen copy of the condition's fields at fire
|
||||
@@ -2991,6 +3031,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
var gate = EvaluateAlarmCommandGate(identity, condition, operation);
|
||||
if (gate is not null) return gate;
|
||||
|
||||
// Gitea #717 owner answer Q4: a Warm Standby refuses every Part 9 condition method before it changes any state
|
||||
// (nothing routed, nothing applied, no event). Unset/None keep today's acceptance.
|
||||
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal($"call {operation}");
|
||||
|
||||
var cmd = new AlarmCommand(
|
||||
AlarmId: condition.NodeId.IdentifierAsString,
|
||||
Operation: operation,
|
||||
@@ -3032,6 +3076,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
var gate = EvaluateAlarmCommandGate(identity, condition, "Acknowledge");
|
||||
if (gate is not null) return gate;
|
||||
|
||||
// Gitea #717 owner answer Q4: refused on a Warm Standby, ahead of (and whatever) the Primary gate below says.
|
||||
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("acknowledge the alarm");
|
||||
|
||||
// PRIMARY GATE, after authorization (a caller who may not ack learns nothing about the
|
||||
// node's role). Only the Primary routes a native ack upstream (DriverHostActor applies the same verdict
|
||||
// to the route), so on any other node the ack is refused outright rather than applied locally and then
|
||||
@@ -3215,6 +3262,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
?? EvaluateEquipmentWriteGate(identity, gatewayWired);
|
||||
if (gate is not null) return gate;
|
||||
|
||||
// Gitea #717 owner answer Q1: a Warm Standby refuses the write synchronously, AFTER authorization (a caller who
|
||||
// may not write learns nothing about the node's role) and before anything reaches the driver. Nothing is
|
||||
// applied, so nothing is reverted. Unset/None never set the flag and keep the #5 optimistic path.
|
||||
if (_standby.RefusesMutations) return WarmStandbyState.StandbyRefusal("write the value");
|
||||
|
||||
// Item A (synchronous structural fail-fast): reject a structurally-invalid write INLINE — return Bad
|
||||
// synchronously so the SDK never applies it (no optimistic-Good-then-revert round-trip + no needless
|
||||
// device dispatch). Runs AFTER the authz gate (so we never leak structure detail to an unauthorised
|
||||
@@ -4120,7 +4172,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
// The SDK exposes the flags separately (no CurrentReadWrite composite): ReadWrite is
|
||||
// CurrentRead | CurrentWrite, with the HistoryRead bit OR-ed in when historized.
|
||||
var access = ComposeAccessLevel(writable, historized, HistoryWritable(_historianDataUpdater));
|
||||
var variable = new EquipmentVariableState(parent, Lock)
|
||||
var variable = new EquipmentVariableState(parent, Lock, _standby)
|
||||
{
|
||||
NodeId = new NodeId(variableNodeId, ns),
|
||||
BrowseName = new QualifiedName(variableNodeId, ns),
|
||||
@@ -5725,6 +5777,88 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
|
||||
lock (Lock) return MonitoredNodes.ContainsKey(nodeId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.7 promotion step 2 / demotion step 5: queues one value to every live, non-stood-down
|
||||
/// ordinary-tag <c>Value</c> item, ignoring its filters (the queue pattern of <see cref="RevalidateMonitoredItems"/>
|
||||
/// — never detach or <c>Disabled</c>, which drops the queue). Promotion queues the current stored value read
|
||||
/// through the item's own IndexRange/DataEncoding; demotion queues <c>BadOutOfService</c>/null.
|
||||
/// </summary>
|
||||
/// <param name="projected">Queue the projected value (demotion) instead of the stored one (promotion).</param>
|
||||
/// <returns>The number of items queued to.</returns>
|
||||
internal int QueueCurrentToLiveItems(bool projected)
|
||||
{
|
||||
var queued = 0;
|
||||
foreach (var (item, tracked) in _liveItems)
|
||||
{
|
||||
if (tracked.Suppressed || tracked.Handle.Node is not EquipmentVariableState variable) continue;
|
||||
if (item is not IDataChangeMonitoredItem2 { AttributeId: Attributes.Value } data) continue;
|
||||
var now = UtcNow;
|
||||
DataValue value;
|
||||
if (projected)
|
||||
{
|
||||
value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, now);
|
||||
}
|
||||
else
|
||||
{
|
||||
var stored = new DataValue();
|
||||
ServiceResult read;
|
||||
lock (Lock) read = variable.ReadAttribute(SystemContext, Attributes.Value, data.IndexRange, data.DataEncoding, ref stored);
|
||||
value = ServiceResult.IsBad(read)
|
||||
? new DataValue(Variant.Null, read.StatusCode, DateTimeUtc.MinValue, now)
|
||||
: new DataValue(stored.WrappedValue, stored.StatusCode, stored.SourceTimestamp, now);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
item.QueueValue(value, StatusCode.IsBad(value.StatusCode) ? new ServiceResult(value.StatusCode) : null, ignoreFilters: true);
|
||||
queued++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A disabled item refuses the value; it re-samples the (projected or not) node when re-enabled.
|
||||
_logger.LogDebug(ex, "OtOpcUaNodeManager: could not queue a Warm transition value to a monitored item");
|
||||
}
|
||||
}
|
||||
|
||||
return queued;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §4.5 promotion refresh: a ConditionRefresh2 on every live event item — Server-object items
|
||||
/// and the direct/folder items of this manager's notifiers — through the SDK's own subscription path, so each
|
||||
/// client sees RefreshStart, the retained conditions (through this manager's ACL decision and the #854/#855
|
||||
/// cut), then RefreshEnd. Must not be called under <c>Lock</c>.
|
||||
/// </summary>
|
||||
/// <returns>The number of event items refreshed.</returns>
|
||||
internal int RefreshAllEventItems()
|
||||
{
|
||||
var items = new List<IEventMonitoredItem>(_allEventsItems.Keys);
|
||||
foreach (var (item, tracked) in _eventItems)
|
||||
if (!tracked.Suppressed) items.Add(item);
|
||||
if (items.Count == 0) return 0;
|
||||
|
||||
var subscriptions = new Dictionary<uint, ISubscription>();
|
||||
foreach (var subscription in Server.SubscriptionManager.GetSubscriptions()) subscriptions[subscription.Id] = subscription;
|
||||
|
||||
var refreshed = 0;
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (!subscriptions.TryGetValue(item.SubscriptionId, out var subscription)) continue;
|
||||
try
|
||||
{
|
||||
subscription.ConditionRefresh2Async(item.Id).AsTask().GetAwaiter().GetResult();
|
||||
refreshed++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A subscription deleted mid-walk, or an item whose mode refuses a refresh: that client re-refreshes itself.
|
||||
_logger.LogDebug(ex, "OtOpcUaNodeManager: Warm promotion refresh of event item {ItemId} failed", item.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return refreshed;
|
||||
}
|
||||
|
||||
/// <summary>Test observability: how many data-change items the revalidation map tracks.</summary>
|
||||
internal int LiveItemCount => _liveItems.Count;
|
||||
|
||||
|
||||
@@ -315,6 +315,160 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
/// </summary>
|
||||
public const byte StartupServiceLevel = 0;
|
||||
|
||||
private OpcUaRedundancyMode _redundancyMode = OpcUaRedundancyMode.Unset;
|
||||
private string _selfUri = string.Empty;
|
||||
private string[] _peerUris = [];
|
||||
private WarmStandbyState _standby = WarmStandbyState.Off;
|
||||
|
||||
/// <summary>The redundancy mode configured before start (Gitea #717).</summary>
|
||||
public OpcUaRedundancyMode RedundancyMode => _redundancyMode;
|
||||
|
||||
/// <summary>The Warm standby flags shared with the node manager.</summary>
|
||||
internal WarmStandbyState Standby => _standby;
|
||||
|
||||
/// <summary>The initial <c>Server.ServiceLevel</c> under <c>Warm</c> (spec §4.3: the NoData band, replacing the
|
||||
/// SDK's 255 and this server's #885 value 0 until the first computed level).</summary>
|
||||
public const byte WarmBootServiceLevel = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Sets the startup-static redundancy metadata (Gitea #717 spec §3) before <c>StartAsync</c>.
|
||||
/// <see cref="OpcUaRedundancyMode.Unset"/> (the default when this is never called) leaves the SDK graph and
|
||||
/// every data path exactly as they are today.
|
||||
/// </summary>
|
||||
/// <param name="mode">The validated mode.</param>
|
||||
/// <param name="effectiveSelfUri">The effective <c>ApplicationConfiguration.ApplicationUri</c>.</param>
|
||||
/// <param name="peerUris">The peer URIs, in configured order.</param>
|
||||
public void ConfigureRedundancy(OpcUaRedundancyMode mode, string effectiveSelfUri, IReadOnlyList<string> peerUris)
|
||||
{
|
||||
if (_otOpcUaNodeManager is not null)
|
||||
throw new InvalidOperationException("Redundancy must be configured before the server starts.");
|
||||
_redundancyMode = mode;
|
||||
_selfUri = effectiveSelfUri;
|
||||
_peerUris = peerUris.ToArray();
|
||||
_standby = new WarmStandbyState(mode == OpcUaRedundancyMode.Warm);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The data-plane transition steps a Warm <c>OpcUaPublishActor</c> drives (spec §4.7), or null when the server
|
||||
/// is not Warm or has not started.
|
||||
/// </summary>
|
||||
/// <returns>The projection, or null.</returns>
|
||||
public IStandbyProjection? CreateStandbyProjection() =>
|
||||
_standby.WarmMode && _otOpcUaNodeManager is { } nm ? new SdkStandbyProjection(nm, _standby) : null;
|
||||
|
||||
/// <summary>Test seam: ids of the live subscriptions, for the decision record's dead-subscription sweep.</summary>
|
||||
private IReadOnlySet<uint> LiveSubscriptionIds()
|
||||
{
|
||||
var ids = new HashSet<uint>();
|
||||
try
|
||||
{
|
||||
if (CurrentInstance?.SubscriptionManager is { } manager)
|
||||
foreach (var subscription in manager.GetSubscriptions()) ids.Add(subscription.Id);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Shutdown race: an empty set only drops records, which re-decide (fail-safe) at their next delivery.
|
||||
}
|
||||
|
||||
return ids;
|
||||
}
|
||||
|
||||
private long _publishesSinceSweep;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask<PublishResponse> PublishAsync(SecureChannelContext secureChannelContext,
|
||||
RequestHeader? requestHeader, ArrayOf<SubscriptionAcknowledgement> subscriptionAcknowledgements,
|
||||
RequestLifetime requestLifetime)
|
||||
{
|
||||
var response = await base.PublishAsync(secureChannelContext, requestHeader, subscriptionAcknowledgements, requestLifetime)
|
||||
.ConfigureAwait(false);
|
||||
if (_standby.Delivery is not { } record || response?.NotificationMessage is not { } message) return response!;
|
||||
|
||||
if (message.NotificationData.Count > 0)
|
||||
{
|
||||
response.NotificationMessage = Decide(record, response.SubscriptionId, message);
|
||||
var available = new List<uint>(response.AvailableSequenceNumbers.Count);
|
||||
for (var i = 0; i < response.AvailableSequenceNumbers.Count; i++) available.Add(response.AvailableSequenceNumbers[i]);
|
||||
record.Prune(response.SubscriptionId, available, message.SequenceNumber);
|
||||
}
|
||||
|
||||
if (Interlocked.Increment(ref _publishesSinceSweep) % 1024 == 0) record.SweepDead(LiveSubscriptionIds());
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask<RepublishResponse> RepublishAsync(SecureChannelContext secureChannelContext,
|
||||
RequestHeader? requestHeader, uint subscriptionId, uint retransmitSequenceNumber, RequestLifetime requestLifetime)
|
||||
{
|
||||
var response = await base.RepublishAsync(secureChannelContext, requestHeader, subscriptionId, retransmitSequenceNumber,
|
||||
requestLifetime).ConfigureAwait(false);
|
||||
if (_standby.Delivery is { } record && response?.NotificationMessage is { NotificationData.Count: > 0 } message)
|
||||
response.NotificationMessage = Decide(record, subscriptionId, message);
|
||||
return response!;
|
||||
}
|
||||
|
||||
/// <summary>The recorded (or, first time, current) decision for one delivery, applied to the SDK's fresh clone.</summary>
|
||||
private NotificationMessage Decide(FirstDeliveryRecord record, uint subscriptionId, NotificationMessage message)
|
||||
{
|
||||
if (!record.Decide(subscriptionId, message.SequenceNumber)) return message;
|
||||
var projected = StandbyNotificationRewriter.Project(message, out var rewritten);
|
||||
if (rewritten > 0)
|
||||
{
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Observability.OtOpcUaTelemetry.StandbyProjectedNotifications.Add(rewritten);
|
||||
}
|
||||
|
||||
return projected;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Installs the standard redundancy metadata (Gitea #717 spec §3, T0a): under <c>Warm</c>
|
||||
/// <c>Server.ServerRedundancy</c> becomes a <c>NonTransparentRedundancyType</c> whose read-only
|
||||
/// <c>ServerUriArray</c> (<c>i=11314</c>) is <c>[self, peers…]</c>, <c>RedundancySupport</c> is <c>Warm</c> and
|
||||
/// ServiceLevel boots at <see cref="WarmBootServiceLevel"/>; under <c>None</c> <c>RedundancySupport</c> is
|
||||
/// <c>None</c> and the SDK's orphan <c>i=11314</c> is deleted, so it is absent by NodeId as well as by browse.
|
||||
/// <c>Unset</c> touches nothing.
|
||||
/// </summary>
|
||||
private async ValueTask InstallRedundancyMetadataAsync(IServerInternal server, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_redundancyMode == OpcUaRedundancyMode.Unset) return;
|
||||
var serverObject = server.ServerObject
|
||||
?? throw new InvalidOperationException("The Server object is unavailable during server startup.");
|
||||
var context = server.DefaultSystemContext;
|
||||
var redundancy = serverObject.ServerRedundancy
|
||||
?? throw new InvalidOperationException("Server.ServerRedundancy is unavailable during server startup.");
|
||||
|
||||
if (_redundancyMode == OpcUaRedundancyMode.None)
|
||||
{
|
||||
redundancy.RedundancySupport!.Value = RedundancySupport.None;
|
||||
redundancy.RedundancySupport.ClearChangeMasks(context, false);
|
||||
// T0a finding: the SDK's predefined i=11314 survives as an orphan (unreferenced, but Good [] by NodeId).
|
||||
if (server.DiagnosticsNodeManager is AsyncCustomNodeManager diagnostics)
|
||||
await diagnostics.DeleteNodeAsync(context, VariableIds.Server_ServerRedundancy_ServerUriArray, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var replacer = server.DiagnosticsNodeManager as IPredefinedNodeSubtypeReplacer
|
||||
?? throw new InvalidOperationException("The diagnostics node manager cannot replace predefined node subtypes.");
|
||||
var replaced = (NonTransparentRedundancyState)await replacer.ReplacePredefinedInstanceSubtypeAsync(context, redundancy,
|
||||
context.CreateInstanceOfNonTransparentRedundancyType(),
|
||||
new Dictionary<QualifiedName, NodeId>
|
||||
{
|
||||
[new QualifiedName(BrowseNames.ServerUriArray, 0)] = VariableIds.Server_ServerRedundancy_ServerUriArray,
|
||||
},
|
||||
node => serverObject.ServerRedundancy = (ServerRedundancyState)node, cancellationToken).ConfigureAwait(false);
|
||||
replaced.RedundancySupport!.Value = RedundancySupport.Warm;
|
||||
replaced.ServerUriArray!.Value = (ArrayOf<string>)new[] { _selfUri }.Concat(_peerUris).ToArray();
|
||||
replaced.ServerUriArray.AccessLevel = replaced.ServerUriArray.UserAccessLevel = AccessLevels.CurrentRead;
|
||||
replaced.ClearChangeMasks(context, true);
|
||||
if (serverObject.ServiceLevel is { } serviceLevel)
|
||||
{
|
||||
serviceLevel.Value = WarmBootServiceLevel;
|
||||
serviceLevel.Timestamp = DateTime.UtcNow;
|
||||
serviceLevel.ClearChangeMasks(context, includeChildren: false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async ValueTask OnNodeManagerStartedAsync(
|
||||
IServerInternal server, CancellationToken cancellationToken = default)
|
||||
@@ -329,6 +483,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
serviceLevel.Timestamp = DateTime.UtcNow;
|
||||
serviceLevel.ClearChangeMasks(server.DefaultSystemContext, includeChildren: false);
|
||||
}
|
||||
await InstallRedundancyMetadataAsync(server, cancellationToken).ConfigureAwait(false);
|
||||
var auditType = server.DiagnosticsNodeManager?
|
||||
.FindPredefinedNode<BaseObjectTypeState>(ObjectTypeIds.AuditWriteUpdateEventType)
|
||||
?? throw new InvalidOperationException("AuditWriteUpdateEventType is unavailable during server startup.");
|
||||
@@ -360,7 +515,7 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
protected override MasterNodeManager CreateMasterNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
{
|
||||
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger);
|
||||
_otOpcUaNodeManager = new OtOpcUaNodeManager(server, configuration, _nodeManagerLogger, _standby);
|
||||
return new MasterNodeManager(server, configuration, dynamicNamespaceUri: null, _otOpcUaNodeManager);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The node-manager-wide Warm standby flags (Gitea #717, spec §4.4–§4.6). One instance per server. Under
|
||||
/// <c>Unset</c>/<c>None</c> <see cref="WarmMode"/> is false and nothing ever sets a flag, so every reader sees
|
||||
/// today's behaviour. Under <c>Warm</c> the node starts Standby (all flags set) until the redundancy layer
|
||||
/// promotes it.
|
||||
/// </summary>
|
||||
internal sealed class WarmStandbyState
|
||||
{
|
||||
private volatile bool _dataProjected;
|
||||
private volatile bool _alarmsSuppressed;
|
||||
|
||||
/// <summary>Initializes the state.</summary>
|
||||
/// <param name="warmMode">Whether this server runs the Warm role contract.</param>
|
||||
public WarmStandbyState(bool warmMode)
|
||||
{
|
||||
WarmMode = warmMode;
|
||||
_dataProjected = warmMode;
|
||||
_alarmsSuppressed = warmMode;
|
||||
Delivery = warmMode ? new FirstDeliveryRecord(standby: true) : null;
|
||||
}
|
||||
|
||||
/// <summary>The not-Warm state (no projection, no suppression, no decision record).</summary>
|
||||
public static WarmStandbyState Off { get; } = new(false);
|
||||
|
||||
/// <summary>Whether this server runs the Warm role contract.</summary>
|
||||
public bool WarmMode { get; }
|
||||
|
||||
/// <summary>Ordinary-tag reads/samples project <c>BadOutOfService</c>; writes and Part 9 methods are refused.</summary>
|
||||
public bool DataProjected
|
||||
{
|
||||
get => _dataProjected;
|
||||
set => _dataProjected = WarmMode && value;
|
||||
}
|
||||
|
||||
/// <summary>New alarm condition events are not delivered (still stamped).</summary>
|
||||
public bool AlarmsSuppressed
|
||||
{
|
||||
get => _alarmsSuppressed;
|
||||
set => _alarmsSuppressed = WarmMode && value;
|
||||
}
|
||||
|
||||
/// <summary>Whether a Warm Standby must refuse a write or a Part 9 method (owner answers Q1/Q4).</summary>
|
||||
public bool RefusesMutations => WarmMode && _dataProjected;
|
||||
|
||||
/// <summary>The first-delivery decision record; null when not Warm.</summary>
|
||||
public FirstDeliveryRecord? Delivery { get; }
|
||||
|
||||
/// <summary>The refusal returned to a write or a Part 9 method on a Warm Standby.</summary>
|
||||
public static ServiceResult StandbyRefusal(string what) => new(StatusCodes.BadInvalidState,
|
||||
new LocalizedText($"This server is the Warm redundancy standby; {what} on the active server (the one with the higher ServiceLevel)."));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ordinary-tag marker (spec §4.6 item 1): set on <see cref="MonitoredItemNotification.Message"/>, a helper
|
||||
/// reference the binary encoder never writes and the SDK's per-delivery <c>CoreUtils.Clone</c> copies
|
||||
/// (pinned by <c>WarmStandbySdkFieldContractTests</c>). Valid only for the notification whose
|
||||
/// <see cref="ClientHandle"/> and <see cref="Value"/> it captured, so a stale marker left on a pooled payload
|
||||
/// (2.0.1's <c>Initialize()</c> does not clear the helper) fails <see cref="Matches"/>.
|
||||
/// </summary>
|
||||
/// <remarks>Derives from <see cref="NotificationMessage"/> only because that is the helper's declared type.</remarks>
|
||||
internal sealed class OrdinaryTagMarker : NotificationMessage
|
||||
{
|
||||
/// <summary>Initializes the marker.</summary>
|
||||
/// <param name="clientHandle">The notification's client handle when it was built.</param>
|
||||
/// <param name="value">The notification's final value when it was built.</param>
|
||||
public OrdinaryTagMarker(uint clientHandle, DataValue value)
|
||||
{
|
||||
ClientHandle = clientHandle;
|
||||
Value = value;
|
||||
}
|
||||
|
||||
/// <summary>The client handle captured at build time.</summary>
|
||||
public uint ClientHandle { get; }
|
||||
|
||||
/// <summary>The value captured at build time.</summary>
|
||||
public DataValue Value { get; }
|
||||
|
||||
/// <summary>Whether <paramref name="n"/> is the notification this marker was made for.</summary>
|
||||
/// <param name="n">A delivered notification.</param>
|
||||
/// <returns>True for a match on client handle and value.</returns>
|
||||
public bool Matches(MonitoredItemNotification n) => n.ClientHandle == ClientHandle && Equals(n.Value, Value);
|
||||
|
||||
/// <summary>Whether <paramref name="n"/> carries a valid ordinary-tag marker.</summary>
|
||||
/// <param name="n">A delivered notification.</param>
|
||||
/// <returns>True when it is an ordinary-tag notification.</returns>
|
||||
public static bool IsOrdinaryTag(MonitoredItemNotification n) => n.Message is OrdinaryTagMarker m && m.Matches(n);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The per-message first-delivery decision (spec §4.6 item 2, D4), keyed by
|
||||
/// <c>(SubscriptionId, SequenceNumber)</c> — never by object: SDK 2.0.1 hands every Publish and Republish a fresh
|
||||
/// clone of the cached message. The first delivery of a sequence number records Active or Standby; every later
|
||||
/// delivery (Republish included) applies the recorded decision. One lock orders a decision against a role flip;
|
||||
/// it is never held across an await or a projection.
|
||||
/// </summary>
|
||||
internal sealed class FirstDeliveryRecord
|
||||
{
|
||||
/// <summary>Recorded messages kept per subscription beyond what <c>AvailableSequenceNumbers</c> prunes.</summary>
|
||||
internal const int MaxPerSubscription = 1024;
|
||||
|
||||
/// <summary>Subscriptions kept before the least-recently-used one is dropped (dead subscriptions).</summary>
|
||||
internal const int MaxSubscriptions = 8192;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<uint, SubscriptionRecord> _subscriptions = new();
|
||||
private bool _standby;
|
||||
private long _touch;
|
||||
|
||||
/// <summary>Initializes the record.</summary>
|
||||
/// <param name="standby">The initial delivery state (a Warm node boots Standby).</param>
|
||||
public FirstDeliveryRecord(bool standby) => _standby = standby;
|
||||
|
||||
/// <summary>Test seam: runs inside the lock after the decision is read and before it is recorded.</summary>
|
||||
internal Action? DecisionBarrier { get; set; }
|
||||
|
||||
/// <summary>The current first-delivery state.</summary>
|
||||
public bool Standby
|
||||
{
|
||||
get { lock (_gate) return _standby; }
|
||||
}
|
||||
|
||||
/// <summary>Flips the first-delivery state (the D4 boundary). Waits for a decision in progress.</summary>
|
||||
/// <param name="standby">Whether later first deliveries are Standby.</param>
|
||||
public void SetStandby(bool standby)
|
||||
{
|
||||
lock (_gate) _standby = standby;
|
||||
}
|
||||
|
||||
/// <summary>The decision for one delivery: the recorded one, or (first delivery) the current state, recorded.</summary>
|
||||
/// <param name="subscriptionId">The subscription.</param>
|
||||
/// <param name="sequenceNumber">The message's sequence number.</param>
|
||||
/// <returns>True when the delivery must be projected (Standby).</returns>
|
||||
public bool Decide(uint subscriptionId, uint sequenceNumber)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_subscriptions.TryGetValue(subscriptionId, out var sub))
|
||||
{
|
||||
if (_subscriptions.Count >= MaxSubscriptions) EvictLeastRecentlyUsed();
|
||||
_subscriptions[subscriptionId] = sub = new SubscriptionRecord();
|
||||
}
|
||||
|
||||
sub.Touched = ++_touch;
|
||||
if (sub.Decisions.TryGetValue(sequenceNumber, out var recorded)) return recorded;
|
||||
var standby = _standby;
|
||||
DecisionBarrier?.Invoke();
|
||||
sub.Decisions[sequenceNumber] = standby;
|
||||
sub.Order.Enqueue(sequenceNumber);
|
||||
while (sub.Order.Count > MaxPerSubscription && sub.Order.TryDequeue(out var oldest)) sub.Decisions.Remove(oldest);
|
||||
return standby;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Drops the decisions of messages the subscription no longer retains (they can never be republished).</summary>
|
||||
/// <param name="subscriptionId">The subscription the Publish response belongs to.</param>
|
||||
/// <param name="available">The response's <c>AvailableSequenceNumbers</c>.</param>
|
||||
/// <param name="justDelivered">The sequence number just delivered (kept: it becomes available for Republish).</param>
|
||||
public void Prune(uint subscriptionId, IReadOnlyCollection<uint> available, uint justDelivered)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_subscriptions.TryGetValue(subscriptionId, out var sub)) return;
|
||||
var keep = new HashSet<uint>(available) { justDelivered };
|
||||
var kept = new Queue<uint>();
|
||||
while (sub.Order.TryDequeue(out var seq))
|
||||
{
|
||||
if (keep.Contains(seq)) kept.Enqueue(seq);
|
||||
else sub.Decisions.Remove(seq);
|
||||
}
|
||||
|
||||
sub.Order.Clear();
|
||||
foreach (var seq in kept) sub.Order.Enqueue(seq);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Drops every record of subscriptions not in <paramref name="live"/>.</summary>
|
||||
/// <param name="live">The ids of live subscriptions.</param>
|
||||
public void SweepDead(IReadOnlySet<uint> live)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (var id in _subscriptions.Keys.Where(id => !live.Contains(id)).ToList()) _subscriptions.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Recorded decisions (test observation).</summary>
|
||||
internal int Count
|
||||
{
|
||||
get { lock (_gate) return _subscriptions.Values.Sum(s => s.Decisions.Count); }
|
||||
}
|
||||
|
||||
/// <summary>Recorded subscriptions (test observation).</summary>
|
||||
internal int SubscriptionCount
|
||||
{
|
||||
get { lock (_gate) return _subscriptions.Count; }
|
||||
}
|
||||
|
||||
private void EvictLeastRecentlyUsed()
|
||||
{
|
||||
var victim = _subscriptions.MinBy(kv => kv.Value.Touched).Key;
|
||||
_subscriptions.Remove(victim);
|
||||
}
|
||||
|
||||
private sealed class SubscriptionRecord
|
||||
{
|
||||
public readonly Dictionary<uint, bool> Decisions = new();
|
||||
public readonly Queue<uint> Order = new();
|
||||
public long Touched;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rewrites a Standby delivery (spec §4.6): every marked ordinary-tag notification whose status is not Bad becomes
|
||||
/// <c>BadOutOfService</c>/null with its server timestamp kept. Builds new objects; the message handed in (the SDK's
|
||||
/// fresh per-delivery clone) and, a fortiori, the SDK's cached copy are never mutated.
|
||||
/// </summary>
|
||||
internal static class StandbyNotificationRewriter
|
||||
{
|
||||
/// <summary>Projects <paramref name="message"/>, or returns it unchanged when nothing needs rewriting.</summary>
|
||||
/// <param name="message">A delivered message.</param>
|
||||
/// <param name="rewritten">The number of notifications rewritten.</param>
|
||||
/// <returns>The message to deliver.</returns>
|
||||
public static NotificationMessage Project(NotificationMessage message, out int rewritten)
|
||||
{
|
||||
rewritten = 0;
|
||||
if (message is null || message.NotificationData.Count == 0) return message!;
|
||||
|
||||
var data = new List<ExtensionObject>(message.NotificationData.Count);
|
||||
var changed = false;
|
||||
for (var i = 0; i < message.NotificationData.Count; i++)
|
||||
{
|
||||
var eo = message.NotificationData[i];
|
||||
if (eo.TryGetValue<DataChangeNotification>(out var dcn) && dcn is not null)
|
||||
{
|
||||
List<MonitoredItemNotification>? items = null;
|
||||
for (var j = 0; j < dcn.MonitoredItems.Count; j++)
|
||||
{
|
||||
var n = dcn.MonitoredItems[j];
|
||||
if (OrdinaryTagMarker.IsOrdinaryTag(n) && !StatusCode.IsBad(n.Value.StatusCode))
|
||||
{
|
||||
items ??= CopyPrefix(dcn, j);
|
||||
items.Add(new MonitoredItemNotification
|
||||
{
|
||||
ClientHandle = n.ClientHandle,
|
||||
Value = new DataValue(Variant.Null, StatusCodes.BadOutOfService, DateTimeUtc.MinValue, n.Value.ServerTimestamp),
|
||||
});
|
||||
rewritten++;
|
||||
}
|
||||
else
|
||||
{
|
||||
items?.Add(n);
|
||||
}
|
||||
}
|
||||
|
||||
if (items is not null)
|
||||
{
|
||||
changed = true;
|
||||
data.Add(new ExtensionObject(new DataChangeNotification { MonitoredItems = items, DiagnosticInfos = dcn.DiagnosticInfos }));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
data.Add(eo);
|
||||
}
|
||||
|
||||
if (!changed) return message;
|
||||
return new NotificationMessage { SequenceNumber = message.SequenceNumber, PublishTime = message.PublishTime, NotificationData = data };
|
||||
}
|
||||
|
||||
private static List<MonitoredItemNotification> CopyPrefix(DataChangeNotification dcn, int count)
|
||||
{
|
||||
var list = new List<MonitoredItemNotification>(dcn.MonitoredItems.Count);
|
||||
for (var k = 0; k < count; k++) list.Add(dcn.MonitoredItems[k]);
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A sampled data-change item for an ordinary-tag <c>Value</c> (spec §4.6 item 1): after the SDK's own per-value
|
||||
/// publish (so the timestamp filter has run) it marks each notification it produced with an
|
||||
/// <see cref="OrdinaryTagMarker"/>. Everything else is the SDK's <see cref="MonitoredItem"/>.
|
||||
/// </summary>
|
||||
internal sealed class OrdinaryTagMonitoredItem : MonitoredItem
|
||||
{
|
||||
/// <summary>Initializes the item exactly as <see cref="MonitoredNodeMonitoredItemManager"/> does.</summary>
|
||||
public OrdinaryTagMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager, NodeHandle handle, uint subscriptionId,
|
||||
uint id, MonitoredItemCreateRequest request, DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn,
|
||||
MonitoringFilter? filter, Opc.Ua.Range? range, double samplingInterval, uint queueSize, bool durable)
|
||||
: base(server, nodeManager, handle, subscriptionId, id, request.ItemToMonitor, diagnosticsMasks, timestampsToReturn,
|
||||
request.MonitoringMode, request.RequestedParameters.ClientHandle, filter, filter, range, samplingInterval, queueSize,
|
||||
request.RequestedParameters.DiscardOldest, 0, durable)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool Publish(OperationContext context, Queue<MonitoredItemNotification> notifications,
|
||||
Queue<DiagnosticInfo> diagnostics, uint maxNotificationsPerPublish, ILogger logger)
|
||||
{
|
||||
var local = new Queue<MonitoredItemNotification>();
|
||||
var localDiagnostics = new Queue<DiagnosticInfo>();
|
||||
var more = base.Publish(context, local, localDiagnostics, maxNotificationsPerPublish, logger);
|
||||
while (local.Count > 0)
|
||||
{
|
||||
var n = local.Dequeue();
|
||||
n.Message = new OrdinaryTagMarker(n.ClientHandle, n.Value);
|
||||
notifications.Enqueue(n);
|
||||
if (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
|
||||
}
|
||||
|
||||
while (localDiagnostics.Count > 0) diagnostics.Enqueue(localDiagnostics.Dequeue());
|
||||
return more;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Installs <see cref="OrdinaryTagMonitoredItem"/> for ordinary-tag (<see cref="EquipmentVariableState"/>)
|
||||
/// <c>Value</c> items only. Re-implements <see cref="IMonitoredItemManager.CreateMonitoredItem"/> on the SDK's
|
||||
/// public, non-sealed manager, so <c>CustomNodeManager2</c>'s <c>is MonitoredNodeMonitoredItemManager</c> checks and
|
||||
/// its internal lifecycle interface keep working (T0c; the 2.0.1 <c>MonitoredItemFactory</c> hook exists only on
|
||||
/// <c>AsyncCustomNodeManager</c>). Installed only under Warm; <c>Unset</c>/<c>None</c> keep the SDK's manager.
|
||||
/// </summary>
|
||||
internal sealed class OrdinaryTagItemManager(IAsyncNodeManager nodeManager, IServerInternal server)
|
||||
: MonitoredNodeMonitoredItemManager(nodeManager, server), IMonitoredItemManager
|
||||
{
|
||||
/// <inheritdoc />
|
||||
ISampledDataChangeMonitoredItem IMonitoredItemManager.CreateMonitoredItem(IServerInternal server, IAsyncNodeManager nodeManager,
|
||||
ServerSystemContext context, NodeHandle handle, uint subscriptionId, double publishingInterval,
|
||||
DiagnosticsMasks diagnosticsMasks, TimestampsToReturn timestampsToReturn, MonitoredItemCreateRequest itemToCreate,
|
||||
Opc.Ua.Range euRange, MonitoringFilter filterToUse, double samplingInterval, uint revisedQueueSize, bool createDurable,
|
||||
MonitoredItemIdFactory monitoredItemIdFactory, Func<ISystemContext, NodeHandle, NodeState, NodeState> addNodeToComponentCache)
|
||||
{
|
||||
if (handle.Node is not EquipmentVariableState || itemToCreate.ItemToMonitor.AttributeId != Attributes.Value)
|
||||
{
|
||||
return CreateMonitoredItem(server, nodeManager, context, handle, subscriptionId, publishingInterval, diagnosticsMasks,
|
||||
timestampsToReturn, itemToCreate, euRange, filterToUse, samplingInterval, revisedQueueSize, createDurable,
|
||||
monitoredItemIdFactory, addNodeToComponentCache);
|
||||
}
|
||||
|
||||
// Mirrors MonitoredNodeMonitoredItemManager.CreateMonitoredItem in exact tagged 2.0.1, swapping only the item type.
|
||||
var cachedNode = addNodeToComponentCache(context, handle, handle.Node);
|
||||
if (!MonitoredNodes.TryGetValue(handle.Node.NodeId, out var monitoredNode))
|
||||
{
|
||||
MonitoredNodes[handle.Node.NodeId] = monitoredNode = new MonitoredNode2(nodeManager, server, cachedNode,
|
||||
nodeManager.IsMultipleEventConsumerNode(cachedNode.NodeId));
|
||||
}
|
||||
|
||||
handle.Node = monitoredNode.Node;
|
||||
handle.MonitoredNode = monitoredNode;
|
||||
uint id;
|
||||
do
|
||||
{
|
||||
id = monitoredItemIdFactory.GetNextId();
|
||||
}
|
||||
while (!MonitoredItems.TryAdd(id, null!));
|
||||
|
||||
var item = new OrdinaryTagMonitoredItem(server, nodeManager, handle, subscriptionId, id, itemToCreate, diagnosticsMasks,
|
||||
timestampsToReturn, filterToUse, euRange, samplingInterval, revisedQueueSize, createDurable);
|
||||
monitoredNode.Add((IDataChangeMonitoredItem2)item);
|
||||
MonitoredItems[id] = item;
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The <see cref="ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection"/> a Warm server hands the redundancy layer.</summary>
|
||||
internal sealed class SdkStandbyProjection(OtOpcUaNodeManager nodeManager, WarmStandbyState state)
|
||||
: ZB.MOM.WW.OtOpcUa.Commons.OpcUa.IStandbyProjection
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public void SetDataProjection(bool projected) => state.DataProjected = projected;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetStandbyDelivery(bool standby) => state.Delivery?.SetStandby(standby);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetAlarmSuppression(bool suppressed) => state.AlarmsSuppressed = suppressed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int QueueCurrentToLiveItems(bool projected) => nodeManager.QueueCurrentToLiveItems(projected);
|
||||
|
||||
/// <inheritdoc />
|
||||
public int RefreshAllEventItems() => nodeManager.RefreshAllEventItems();
|
||||
}
|
||||
@@ -258,6 +258,52 @@ public static class DeploymentArtifact
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §3: the <c>ServerCluster.RedundancyMode</c> of this node's cluster as the artifact carries it
|
||||
/// (<c>None</c>/<c>Warm</c>/<c>Hot</c>), for the deploy-time cross-check against <c>OpcUa:Redundancy:Mode</c>.
|
||||
/// A single-cluster artifact answers for its only cluster; a multi-cluster one for the cluster of this node's
|
||||
/// <c>ClusterNode</c> row. Null when the artifact does not say (empty, malformed, no matching cluster).
|
||||
/// </summary>
|
||||
/// <param name="blob">The deployment artifact.</param>
|
||||
/// <param name="nodeId">This node's <c>host:port</c> id.</param>
|
||||
/// <returns>The mode name, or null.</returns>
|
||||
public static string? ReadClusterRedundancyMode(ReadOnlySpan<byte> blob, string nodeId)
|
||||
{
|
||||
if (blob.IsEmpty) return null;
|
||||
try
|
||||
{
|
||||
using var parsed = Open(blob.ToArray());
|
||||
var root = parsed.Root;
|
||||
if (!root.TryGetProperty("Clusters", out var clusters) || clusters.ValueKind != JsonValueKind.Array) return null;
|
||||
var scope = parsed.ResolveClusterScope(nodeId);
|
||||
JsonElement? match = null;
|
||||
foreach (var cluster in clusters.EnumerateArray())
|
||||
{
|
||||
if (cluster.ValueKind != JsonValueKind.Object) continue;
|
||||
if (scope.Mode == ClusterFilterMode.None) { match = cluster; break; }
|
||||
if (scope.Mode == ClusterFilterMode.ScopeTo
|
||||
&& string.Equals(cluster.TryGetProperty("ClusterId", out var cid) && cid.ValueKind == JsonValueKind.String ? cid.GetString() : null, scope.ClusterId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
match = cluster;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (match is not { } m || !m.TryGetProperty("RedundancyMode", out var mode)) return null;
|
||||
return mode.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => mode.GetString(),
|
||||
JsonValueKind.Number when mode.TryGetInt32(out var n) && Enum.IsDefined(typeof(Configuration.Enums.RedundancyMode), n)
|
||||
=> ((Configuration.Enums.RedundancyMode)n).ToString(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolve how a node should scope a deployment artifact to its own ClusterId. Single-cluster
|
||||
/// (or legacy) artifacts resolve to <see cref="ClusterFilterMode.None"/> so every existing
|
||||
|
||||
@@ -789,7 +789,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
TimeProvider? writeClock = null,
|
||||
IScriptRunner? scriptedAlarmRunner = null,
|
||||
TimeSpan? childShutdownTimeout = null) =>
|
||||
TimeSpan? childShutdownTimeout = null,
|
||||
bool enforceSnapshotOrdering = false) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -803,7 +804,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
|
||||
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout));
|
||||
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout,
|
||||
enforceSnapshotOrdering));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -938,8 +940,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
TimeProvider? writeClock = null,
|
||||
IScriptRunner? scriptedAlarmRunner = null,
|
||||
TimeSpan? childShutdownTimeout = null)
|
||||
TimeSpan? childShutdownTimeout = null,
|
||||
bool enforceSnapshotOrdering = false)
|
||||
{
|
||||
_snapshotOrdering = enforceSnapshotOrdering ? new SnapshotRevisionGate() : null;
|
||||
_childShutdownTimeout = childShutdownTimeout;
|
||||
_scriptedAlarmRunner = scriptedAlarmRunner;
|
||||
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
|
||||
@@ -2552,8 +2556,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// the cached role unchanged; when this node has a real driver peer (count > 1) that omission is the
|
||||
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
|
||||
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
|
||||
/// <summary>Gitea #717: the snapshot order gate under Warm; null (no ordering, today's behaviour) otherwise.</summary>
|
||||
private readonly SnapshotRevisionGate? _snapshotOrdering;
|
||||
|
||||
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
// Gitea #717 spec §4.2, under OpcUa:Redundancy:Mode=Warm only (Unset/None keep today's unconditional replace):
|
||||
// a snapshot older than the last accepted one — a late heartbeat of an old singleton incarnation — is dropped,
|
||||
// so it cannot overwrite the new incarnation's roles. An unrevisioned snapshot (pre-#717 publisher) still feeds
|
||||
// the gates, as before.
|
||||
if (_snapshotOrdering?.Offer(msg) == SnapshotRejection.OlderRevision)
|
||||
{
|
||||
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
|
||||
return;
|
||||
}
|
||||
|
||||
_redundancySnapshotReceived = true;
|
||||
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
|
||||
if (local is not null)
|
||||
|
||||
@@ -14,6 +14,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
|
||||
@@ -326,7 +327,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null) =>
|
||||
IOpcUaEndpointState? endpointState = null,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
@@ -341,7 +344,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
dbLess,
|
||||
dataPlaneAuthority,
|
||||
utcNow,
|
||||
endpointState)).WithDispatcher(DispatcherId);
|
||||
endpointState,
|
||||
redundancy,
|
||||
standbyProjection)).WithDispatcher(DispatcherId);
|
||||
|
||||
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
|
||||
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
|
||||
@@ -385,7 +390,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null) =>
|
||||
IOpcUaEndpointState? endpointState = null,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
@@ -400,7 +407,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
dbLess,
|
||||
dataPlaneAuthority,
|
||||
utcNow,
|
||||
endpointState));
|
||||
endpointState,
|
||||
redundancy,
|
||||
standbyProjection));
|
||||
|
||||
/// <summary>Test-only Props exposing the deferred-write bound and the plan-step seams.</summary>
|
||||
internal static Props PropsForTests(
|
||||
@@ -461,10 +470,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null)
|
||||
IOpcUaEndpointState? endpointState = null,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null)
|
||||
: this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow,
|
||||
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null,
|
||||
endpointState)
|
||||
endpointState, redundancy, standbyProjection)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -488,8 +499,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Func<DateTime>? utcNow,
|
||||
int deferredWriteCapacity,
|
||||
TestHooks? hooks,
|
||||
IOpcUaEndpointState? endpointState)
|
||||
IOpcUaEndpointState? endpointState,
|
||||
WarmRedundancySettings? redundancy = null,
|
||||
IStandbyProjection? standbyProjection = null)
|
||||
{
|
||||
_redundancy = redundancy ?? WarmRedundancySettings.Unset;
|
||||
_standbyProjection = standbyProjection;
|
||||
_deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity));
|
||||
_hooks = hooks;
|
||||
_sink = sink;
|
||||
@@ -523,6 +538,12 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Receive<HealthTick>(_ => OnHealthTick());
|
||||
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(HandlePeerProbe);
|
||||
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
|
||||
// Gitea #717 spec §4.1 / owner answer Q2: under Warm the verdict re-derives on membership and reachability, not
|
||||
// only on snapshots. Nothing subscribes these under Unset/None.
|
||||
Receive<ClusterEvent.IMemberEvent>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.IReachabilityEvent>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.CurrentClusterState>(_ => RecomputeServiceLevel());
|
||||
Receive<WarmRetry>(_ => RecomputeServiceLevel());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -543,6 +564,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
base.PreRestart(reason, message);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
try { _cluster.Unsubscribe(Self); }
|
||||
catch (Exception) { /* the cluster is shutting down with us */ }
|
||||
}
|
||||
|
||||
base.PostStop();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
@@ -551,6 +584,20 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
|
||||
}
|
||||
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
OtOpcUaTelemetry.SetWarmRole(0);
|
||||
try
|
||||
{
|
||||
_cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents,
|
||||
typeof(ClusterEvent.IMemberEvent), typeof(ClusterEvent.IReachabilityEvent));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "OpcUaPublish: Warm redundancy could not subscribe to cluster membership; the verdict re-derives on snapshots only");
|
||||
}
|
||||
}
|
||||
|
||||
// DB-backed nodes refresh their cached probe status; DB-less nodes still need periodic
|
||||
// snapshot/probe-verdict age checks even when no new messages arrive. Legacy nodes keep
|
||||
// their role-only behavior without a timer.
|
||||
@@ -672,7 +719,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleRebuild(RebuildAddressSpace msg) => StartRebuild(msg, Sender);
|
||||
private void HandleRebuild(RebuildAddressSpace msg)
|
||||
{
|
||||
if (msg.Artifact is { Length: > 0 } artifact) CheckRedundancyMode(artifact);
|
||||
StartRebuild(msg, Sender);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The entry half of a rebuild, on the actor thread. The raw-sink fallback (no applier) and the
|
||||
@@ -1465,26 +1516,34 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleServiceLevelChanged(ServiceLevelChanged msg)
|
||||
private void HandleServiceLevelChanged(ServiceLevelChanged msg) => TryPublishServiceLevel(msg.ServiceLevel);
|
||||
|
||||
/// <summary>The dedup/publish/metric step behind <see cref="ServiceLevelChanged"/>; synchronous so a Warm
|
||||
/// transition can order it against the projection steps (spec §4.7).</summary>
|
||||
/// <param name="level">The level to publish.</param>
|
||||
/// <returns>True when the level is published (or already was).</returns>
|
||||
private bool TryPublishServiceLevel(byte level)
|
||||
{
|
||||
// Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a
|
||||
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
|
||||
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
|
||||
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
|
||||
if (_publishedAtLeastOnce && level == _lastServiceLevel) return true;
|
||||
try
|
||||
{
|
||||
_serviceLevel.Publish(msg.ServiceLevel);
|
||||
_lastServiceLevel = msg.ServiceLevel;
|
||||
_serviceLevel.Publish(level);
|
||||
_lastServiceLevel = level;
|
||||
_publishedAtLeastOnce = true;
|
||||
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
|
||||
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
|
||||
_log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel);
|
||||
new KeyValuePair<string, object?>("level", level));
|
||||
_log.Debug("OpcUaPublish: ServiceLevel={Level}", level);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failed write may be partially applied; even a previous successful value must retry.
|
||||
_publishedAtLeastOnce = false;
|
||||
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", msg.ServiceLevel);
|
||||
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", level);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1494,6 +1553,28 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// wired) or via the legacy role-only seam (back-compat / bootstrap).</summary>
|
||||
private void HandleRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
// Gitea #717 spec §4.2: under Warm a snapshot older than the last accepted one (a late heartbeat from an old
|
||||
// singleton incarnation) is dropped outright. An unrevisioned one (a pre-#717 publisher mid-upgrade) still
|
||||
// feeds the health inputs but can never make this node Active. Unset/None keep today's unconditional
|
||||
// replace below.
|
||||
switch (_revisionGate.Offer(msg))
|
||||
{
|
||||
case SnapshotRejection.OlderRevision:
|
||||
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "older_revision"));
|
||||
_log.Debug("OpcUaPublish: dropped redundancy snapshot {Revision} (last accepted {Last})",
|
||||
msg.Revision, _revisionGate.LastAccepted);
|
||||
return;
|
||||
case SnapshotRejection.Unrevisioned:
|
||||
OtOpcUaTelemetry.SnapshotRejected.Add(1, new KeyValuePair<string, object?>("reason", "unrevisioned"));
|
||||
break;
|
||||
default:
|
||||
_acceptedSnapshot = msg;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_lastSnapshot = msg;
|
||||
RecomputeServiceLevel();
|
||||
}
|
||||
@@ -1576,6 +1657,11 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
private void RecomputeServiceLevel()
|
||||
{
|
||||
if (_localNode is null || _lastSnapshot is null) return;
|
||||
if (_redundancy.IsWarm)
|
||||
{
|
||||
RecomputeWarm();
|
||||
return;
|
||||
}
|
||||
|
||||
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
|
||||
|
||||
@@ -1635,6 +1721,197 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Warm redundancy (Gitea #717)
|
||||
|
||||
private sealed class WarmRetry { public static readonly WarmRetry Instance = new(); private WarmRetry() { } }
|
||||
|
||||
private readonly WarmRedundancySettings _redundancy;
|
||||
private readonly IStandbyProjection? _standbyProjection;
|
||||
private readonly SnapshotRevisionGate _revisionGate = new();
|
||||
/// <summary>The latest ACCEPTED revisioned snapshot — the only one the Warm verdict reads (spec §4.2).</summary>
|
||||
private RedundancyStateChanged? _acceptedSnapshot;
|
||||
/// <summary>The role the data plane is in. A Warm node manager boots Standby, so this starts Standby.</summary>
|
||||
private WarmRole _warmRole = WarmRole.Standby;
|
||||
/// <summary>A promotion whose Active ServiceLevel publish failed (spec §4.7): the data plane is Active and the
|
||||
/// level is retried; the Standby level stays visible meanwhile, never the reverse.</summary>
|
||||
private bool _promotionLevelPending;
|
||||
private DateTime? _disagreeSinceUtc;
|
||||
private bool _disagreementWarned;
|
||||
|
||||
/// <summary>How long membership and snapshot may disagree before a Warning (two heartbeats, spec §4.1).</summary>
|
||||
internal static readonly TimeSpan WarmDisagreementWarning = TimeSpan.FromSeconds(20);
|
||||
|
||||
/// <summary>The current Warm role (test observation).</summary>
|
||||
internal WarmRole CurrentWarmRole => _warmRole;
|
||||
|
||||
/// <summary>A promoted node whose Active ServiceLevel publish is being retried (test observation).</summary>
|
||||
internal bool PromotionLevelPending => _promotionLevelPending;
|
||||
|
||||
/// <summary>
|
||||
/// The Warm ServiceLevel + role path (spec §4.1, §4.3, §4.7): the health facts the non-Warm paths use, the
|
||||
/// Warm verdict, then a transition in the fixed step order when the verdict changed, else just the band.
|
||||
/// </summary>
|
||||
private void RecomputeWarm()
|
||||
{
|
||||
var self = _localNode!.Value;
|
||||
var entry = _lastSnapshot!.Nodes.FirstOrDefault(n => n.NodeId == self);
|
||||
var now = _utcNow();
|
||||
var memberState = SafeSelfStatus();
|
||||
var listening = EndpointListening();
|
||||
|
||||
bool dbReachable;
|
||||
bool stale;
|
||||
if (_dbLess || _dbHealthProbe is null || _lastDbHealth is null)
|
||||
{
|
||||
dbReachable = true;
|
||||
stale = entry is null || (now - entry.AsOfUtc) > _staleWindow;
|
||||
}
|
||||
else
|
||||
{
|
||||
dbReachable = _lastDbHealth.Reachable;
|
||||
stale = !_lastDbHealth.Reachable
|
||||
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|
||||
|| entry is null || (now - entry.AsOfUtc) > _staleWindow;
|
||||
}
|
||||
|
||||
var verdict = EvaluateWarmVerdict(self, memberState, listening);
|
||||
TrackDisagreement(verdict, now);
|
||||
|
||||
var detached = entry is null || entry.Role == RedundancyRole.Detached;
|
||||
var inputs = new NodeHealthInputs(memberState, dbReachable, OpcUaProbeOk(), stale, IsDriverPrimary: false, listening);
|
||||
var target = detached ? WarmRole.Standby : verdict.Role;
|
||||
var level = detached ? (byte)0 : ServiceLevelCalculator.ComputeWarm(inputs, target == WarmRole.Active);
|
||||
ApplyWarmTransition(target, level, verdict.Reason);
|
||||
}
|
||||
|
||||
private WarmVerdict EvaluateWarmVerdict(NodeId self, MemberStatus memberState, bool listening)
|
||||
{
|
||||
NodeId? oldest = null, oldestReachable = null;
|
||||
var primaryUnreachable = false;
|
||||
var snapshotPrimary = _acceptedSnapshot?.Nodes.FirstOrDefault(n => n.Role == RedundancyRole.Primary)?.NodeId;
|
||||
try
|
||||
{
|
||||
var state = _cluster.State;
|
||||
var unreachable = state.Unreachable.ToHashSet();
|
||||
oldest = WarmRoleAuthority.OldestUpMember(state.Members, "driver");
|
||||
oldestReachable = WarmRoleAuthority.OldestUpMember(state.Members, "driver", unreachable);
|
||||
primaryUnreachable = snapshotPrimary is { } sp && unreachable.Any(m => WarmRoleAuthority.ToNodeId(m) == sp);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "OpcUaPublish: cluster state unavailable for the Warm verdict; treating membership as unknown");
|
||||
}
|
||||
|
||||
return WarmRoleAuthority.Evaluate(new WarmRoleInputs(
|
||||
self, memberState == MemberStatus.Up, listening, oldest, oldestReachable,
|
||||
HasAcceptedSnapshot: _acceptedSnapshot is not null, snapshotPrimary, primaryUnreachable,
|
||||
_redundancy.PromoteWhilePrimaryUnreachable));
|
||||
}
|
||||
|
||||
private void TrackDisagreement(WarmVerdict verdict, DateTime now)
|
||||
{
|
||||
if (!verdict.Disagreement)
|
||||
{
|
||||
_disagreeSinceUtc = null;
|
||||
_disagreementWarned = false;
|
||||
return;
|
||||
}
|
||||
|
||||
_disagreeSinceUtc ??= now;
|
||||
if (_disagreementWarned || now - _disagreeSinceUtc.Value <= WarmDisagreementWarning) return;
|
||||
_disagreementWarned = true;
|
||||
_log.Warning(
|
||||
"OpcUaPublish: Warm redundancy — cluster membership and the redundancy snapshot (revision {Revision}) have disagreed about this node being Primary for more than {Seconds}s; the node stays {Role}",
|
||||
_revisionGate.LastAccepted, WarmDisagreementWarning.TotalSeconds, _warmRole);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the data plane to <paramref name="target"/> in the spec §4.7 order and publishes
|
||||
/// <paramref name="level"/>. Invariant: an Active-band ServiceLevel is never visible while the projection is
|
||||
/// closed — a promotion publishes the level LAST and, if that publish fails, retries it; a demotion publishes
|
||||
/// the Standby level FIRST and degrades the data whether or not that publish succeeded (failing safe).
|
||||
/// </summary>
|
||||
private void ApplyWarmTransition(WarmRole target, byte level, string reason)
|
||||
{
|
||||
if (target == _warmRole)
|
||||
{
|
||||
if (!TryPublishServiceLevel(level) && target == WarmRole.Active) ScheduleWarmRetry();
|
||||
else if (target == WarmRole.Active) _promotionLevelPending = false;
|
||||
return;
|
||||
}
|
||||
|
||||
var projection = _standbyProjection;
|
||||
if (target == WarmRole.Active)
|
||||
{
|
||||
projection?.SetDataProjection(false); // 1. open the read projection
|
||||
projection?.QueueCurrentToLiveItems(false); // 2. current Good value to every live data item
|
||||
projection?.SetStandbyDelivery(false); // 3. first deliveries decided Active
|
||||
projection?.SetAlarmSuppression(false); // 4. new alarm events delivered
|
||||
projection?.RefreshAllEventItems(); // 5. promotion refresh on every live event item
|
||||
_warmRole = WarmRole.Active;
|
||||
if (TryPublishServiceLevel(level)) // 6. Active ServiceLevel, last
|
||||
{
|
||||
_promotionLevelPending = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
_promotionLevelPending = true;
|
||||
ScheduleWarmRetry();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
TryPublishServiceLevel(level); // 1. Standby ServiceLevel, first (continue on failure)
|
||||
projection?.SetStandbyDelivery(true); // 2. the D4 boundary
|
||||
projection?.SetAlarmSuppression(true); // 3. suppress new alarm events
|
||||
projection?.SetDataProjection(true); // 4. close the read projection
|
||||
projection?.QueueCurrentToLiveItems(true); // 5. BadOutOfService to every live data item
|
||||
_warmRole = WarmRole.Standby;
|
||||
_promotionLevelPending = false;
|
||||
}
|
||||
|
||||
OtOpcUaTelemetry.SetWarmRole(_warmRole == WarmRole.Active ? 1 : 0);
|
||||
OtOpcUaTelemetry.WarmTransition.Add(1,
|
||||
new KeyValuePair<string, object?>("direction", target == WarmRole.Active ? "promote" : "demote"),
|
||||
new KeyValuePair<string, object?>("reason", reason));
|
||||
_log.Info(
|
||||
"OpcUaPublish: Warm redundancy {Direction} to {Role} (reason={Reason}, revision={Revision}, ServiceLevel={Level})",
|
||||
target == WarmRole.Active ? "promotion" : "demotion", _warmRole, reason, _revisionGate.LastAccepted, level);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #717 spec §3 deploy-time cross-check: under <c>None</c> or <c>Warm</c>, an artifact whose cluster
|
||||
/// <c>RedundancyMode</c> disagrees with the local mode (<c>Hot</c> included, owner answer Q5) logs an Error and
|
||||
/// counts <c>otopcua.redundancy.mode_mismatch</c>. The node keeps its local mode — the only value available
|
||||
/// before SDK start. <c>Unset</c> has opted out and is not checked.
|
||||
/// </summary>
|
||||
/// <param name="artifact">The applied artifact bytes.</param>
|
||||
private void CheckRedundancyMode(byte[] artifact)
|
||||
{
|
||||
if (_redundancy.Mode == OpcUaRedundancyMode.Unset || _localNode is null) return;
|
||||
string? clusterMode;
|
||||
try
|
||||
{
|
||||
clusterMode = DeploymentArtifact.ReadClusterRedundancyMode(artifact, _localNode.Value.Value);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "OpcUaPublish: could not read the artifact's cluster RedundancyMode");
|
||||
return;
|
||||
}
|
||||
|
||||
if (clusterMode is null) return;
|
||||
if (string.Equals(clusterMode, _redundancy.Mode.ToString(), StringComparison.OrdinalIgnoreCase)) return;
|
||||
OtOpcUaTelemetry.RedundancyModeMismatch.Add(1,
|
||||
new KeyValuePair<string, object?>("local", _redundancy.Mode.ToString()),
|
||||
new KeyValuePair<string, object?>("cluster", clusterMode));
|
||||
_log.Error(
|
||||
"OpcUaPublish: the deployed ServerCluster.RedundancyMode for this node's cluster is {ClusterMode} but OpcUa:Redundancy:Mode is {LocalMode}; the node keeps running as {LocalMode} (local configuration is the authority — fix whichever is wrong)",
|
||||
clusterMode, _redundancy.Mode, _redundancy.Mode);
|
||||
}
|
||||
|
||||
private void ScheduleWarmRetry() => Timers.StartSingleTimer("warm-retry", WarmRetry.Instance, TimeSpan.FromSeconds(1));
|
||||
|
||||
/// <summary>The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200,
|
||||
/// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
|
||||
private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
using Akka.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
|
||||
/// <summary>The Warm role (Gitea #717 spec §4.1).</summary>
|
||||
public enum WarmRole
|
||||
{
|
||||
/// <summary>Degraded standby: projected data, suppressed new alarm events, Degraded-band ServiceLevel.</summary>
|
||||
Standby,
|
||||
|
||||
/// <summary>Serves data and alarms; Healthy-band ServiceLevel.</summary>
|
||||
Active,
|
||||
}
|
||||
|
||||
/// <summary>The facts the Warm verdict is a pure function of.</summary>
|
||||
/// <param name="Self">This node's canonical <c>host:port</c> id.</param>
|
||||
/// <param name="SelfUp">This node's own member status is Up.</param>
|
||||
/// <param name="EndpointListening">This node's OPC UA endpoint did not fail to start (#762).</param>
|
||||
/// <param name="OldestUpDriver">The oldest Up <c>driver</c> member in this node's OWN cluster state (the singleton
|
||||
/// placement rule), or null when there is none.</param>
|
||||
/// <param name="OldestReachableUpDriver">The same, ignoring members this node sees as unreachable.</param>
|
||||
/// <param name="HasAcceptedSnapshot">A revisioned snapshot has been accepted (spec §4.2). Unrevisioned snapshots
|
||||
/// never count.</param>
|
||||
/// <param name="SnapshotPrimary">The Primary named by the latest accepted revisioned snapshot, or null.</param>
|
||||
/// <param name="SnapshotPrimaryUnreachable">This node sees <paramref name="SnapshotPrimary"/> as unreachable.</param>
|
||||
/// <param name="PromoteWhilePrimaryUnreachable">The owner-answer-Q2 option (default on).</param>
|
||||
public readonly record struct WarmRoleInputs(
|
||||
NodeId Self,
|
||||
bool SelfUp,
|
||||
bool EndpointListening,
|
||||
NodeId? OldestUpDriver,
|
||||
NodeId? OldestReachableUpDriver,
|
||||
bool HasAcceptedSnapshot,
|
||||
NodeId? SnapshotPrimary,
|
||||
bool SnapshotPrimaryUnreachable,
|
||||
bool PromoteWhilePrimaryUnreachable);
|
||||
|
||||
/// <summary>The verdict and why.</summary>
|
||||
/// <param name="Role">Active or Standby.</param>
|
||||
/// <param name="Reason">A short reason for the transition metric/log: <c>boot</c>, <c>endpoint</c>,
|
||||
/// <c>membership</c>, <c>snapshot</c>, <c>unreachable-primary</c> or <c>agreed</c>.</param>
|
||||
/// <param name="Disagreement">Membership (2) and snapshot (3) disagree about whether this node is Primary.</param>
|
||||
public readonly record struct WarmVerdict(WarmRole Role, string Reason, bool Disagreement);
|
||||
|
||||
/// <summary>
|
||||
/// The Warm role verdict (Gitea #717 spec §4.1), a pure function so every row is unit-testable. Active iff the
|
||||
/// node is Up and listening, its own cluster state names it the oldest Up driver member, and the latest accepted
|
||||
/// revisioned snapshot names it Primary. Anything else — no snapshot yet, an unrevisioned snapshot, a
|
||||
/// disagreement — is Standby. With <see cref="WarmRoleInputs.PromoteWhilePrimaryUnreachable"/> the survivor of a
|
||||
/// crashed Primary goes Active once that Primary is unreachable, before <c>auto-down</c> removes it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is a separate output from <c>PrimaryGatePolicy</c>, whose lone-node allow is unchanged for writes and
|
||||
/// acks (D7).
|
||||
/// </remarks>
|
||||
public static class WarmRoleAuthority
|
||||
{
|
||||
/// <summary>Evaluates the verdict.</summary>
|
||||
/// <param name="i">The inputs.</param>
|
||||
/// <returns>The verdict.</returns>
|
||||
public static WarmVerdict Evaluate(in WarmRoleInputs i)
|
||||
{
|
||||
if (!i.SelfUp) return new(WarmRole.Standby, "membership", false);
|
||||
if (!i.EndpointListening) return new(WarmRole.Standby, "endpoint", false);
|
||||
if (!i.HasAcceptedSnapshot) return new(WarmRole.Standby, "boot", false);
|
||||
|
||||
var membershipSaysSelf = i.OldestUpDriver == i.Self;
|
||||
var snapshotSaysSelf = i.SnapshotPrimary == i.Self;
|
||||
var disagreement = membershipSaysSelf != snapshotSaysSelf;
|
||||
|
||||
if (membershipSaysSelf && snapshotSaysSelf) return new(WarmRole.Active, "agreed", false);
|
||||
|
||||
// Owner answer Q2: a SIGKILLed Primary stays an Up-but-unreachable member until auto-down removes it, and the
|
||||
// singleton usually died with it. Evaluate (2) and (3) with unreachable members excluded.
|
||||
if (i.PromoteWhilePrimaryUnreachable
|
||||
&& i.OldestReachableUpDriver == i.Self
|
||||
&& (snapshotSaysSelf || (i.SnapshotPrimary is not null && i.SnapshotPrimaryUnreachable)))
|
||||
{
|
||||
return new(WarmRole.Active, "unreachable-primary", disagreement);
|
||||
}
|
||||
|
||||
return new(WarmRole.Standby, membershipSaysSelf ? "snapshot" : "membership", disagreement);
|
||||
}
|
||||
|
||||
/// <summary>The oldest Up member carrying <paramref name="role"/> (the singleton placement rule), optionally
|
||||
/// ignoring <paramref name="unreachable"/> members.</summary>
|
||||
/// <param name="members">The current members.</param>
|
||||
/// <param name="role">The role.</param>
|
||||
/// <param name="unreachable">Members to ignore, or null.</param>
|
||||
/// <returns>Its canonical node id, or null.</returns>
|
||||
public static NodeId? OldestUpMember(IEnumerable<Member> members, string role, IReadOnlySet<Member>? unreachable = null)
|
||||
{
|
||||
Member? oldest = null;
|
||||
foreach (var m in members)
|
||||
{
|
||||
if (m.Status != MemberStatus.Up || !m.HasRole(role)) continue;
|
||||
if (unreachable is not null && unreachable.Contains(m)) continue;
|
||||
if (oldest is null || m.IsOlderThan(oldest)) oldest = m;
|
||||
}
|
||||
|
||||
return oldest is null ? null : ToNodeId(oldest);
|
||||
}
|
||||
|
||||
/// <summary>The canonical <c>host:port</c> id of a member (the RedundancyStateActor format).</summary>
|
||||
/// <param name="m">The member.</param>
|
||||
/// <returns>The node id.</returns>
|
||||
public static NodeId ToNodeId(Member m) => NodeId.Parse($"{m.Address.Host ?? string.Empty}:{m.Address.Port ?? 0}");
|
||||
}
|
||||
@@ -566,6 +566,9 @@ public static class ServiceCollectionExtensions
|
||||
// Optional: a harness that builds this graph without the hosted service registers none, and both
|
||||
// consumers then behave as if the endpoint were listening (the pre-#762 behaviour).
|
||||
var endpointState = resolver.GetService<IOpcUaEndpointState>();
|
||||
// Gitea #717: OpcUa:Redundancy (registered by the Host from the validated OpcUa options). Absent in harnesses ⇒
|
||||
// Unset ⇒ today's behaviour. The standby projection is late-bound like the ServiceLevel publisher.
|
||||
var redundancy = resolver.GetService<WarmRedundancySettings>() ?? WarmRedundancySettings.Unset;
|
||||
var publishActor = system.ActorOf(
|
||||
OpcUaPublishActor.Props(
|
||||
sink: addressSpaceSink,
|
||||
@@ -576,7 +579,9 @@ public static class ServiceCollectionExtensions
|
||||
dbHealthProbe: dbHealth,
|
||||
dbLess: dbFactory is null,
|
||||
dataPlaneAuthority: dataPlaneAuthority,
|
||||
endpointState: endpointState),
|
||||
endpointState: endpointState,
|
||||
redundancy: redundancy,
|
||||
standbyProjection: redundancy.IsWarm ? resolver.GetService<DeferredStandbyProjection>() : null),
|
||||
OpcUaPublishActorName);
|
||||
registry.Register<OpcUaPublishActorKey>(publishActor);
|
||||
|
||||
@@ -670,7 +675,9 @@ public static class ServiceCollectionExtensions
|
||||
// lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
|
||||
// the InProcess default, when the seam is not listed, and in test graphs that register no provider.
|
||||
scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
|
||||
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
|
||||
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm),
|
||||
// Gitea #717 spec §4.2: drop older redundancy snapshots under Warm only.
|
||||
enforceSnapshotOrdering: redundancy.IsWarm),
|
||||
DriverHostActorName);
|
||||
registry.Register<DriverHostActorKey>(driverHost);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user