using System.Globalization; using Akka.Actor; using Akka.Event; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol; using ZB.MOM.WW.ScadaBridge.Commons.Serialization; using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections; namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors; /// /// ScadaBridge#138 — ServiceLevel-aware failover between a primary and a backup endpoint. /// /// /// Why. Until now the connection failed over only when the link itself misbehaved /// (FailoverRetryCount consecutive connect failures or unstable disconnects). Under OtOpcUa /// Warm redundancy (lmxopcua#717) the standby keeps a perfectly healthy session but serves /// BadOutOfService data at a standby ServiceLevel (90 healthy / 50 degraded / 1 booting), so a /// connection that lands there would sit on Bad quality indefinitely. /// /// /// /// What. While Connected, the connected endpoint's Server_ServiceLevel is polled every /// . A level below /// (200, the bottom of the Part 4 Healthy band) starts a /// degraded streak; any Healthy or unreadable sample ends it. Once the streak has lasted the /// connection's degraded window — or immediately on the first sample after a (re)connect, which is /// the "prefer the better endpoint on connect" rule — the other endpoint is probed with a /// short-lived session. The connection switches only when that probe reads a Healthy level that is /// also higher than the connected one, and never within the dwell time of the previous /// ServiceLevel switch. Otherwise it stays put and the window re-arms. /// /// /// /// Unchanged behaviour. A level of 200 or more never starts a streak, so today's OtOpcUa pair /// (250 Primary / 240 Secondary) never triggers anything; an adapter without /// (MxGateway) and a connection without a backup are never /// monitored at all. The switch itself reuses and the reconnect /// cycle, exactly like the connection-driven failovers. /// /// /// /// A server going away (ScadaBridge#141). A graceful stop of a redundant OtOpcUa node reports /// ServiceLevel 0 for only a few seconds before the session closes — shorter than the degraded window /// — and the following unreadable sample cleared the streak, so the connection fell back to the /// connectivity path and retried the stopped node FailoverRetryCount times (66 s on QA). Two /// going-away triggers now skip that wait: (1) a sample of (0, /// Part 4 "Maintenance": the server is not available for use) probes the other endpoint at once, /// without the window; and (2) when the link is lost while the last KNOWN level of the session was /// Degraded (below 200), the other endpoint is probed before the first reconnect attempt and, if it /// answers Healthy, the connection switches straight to it instead of spending the retry count on the /// endpoint that announced it was degraded. Both keep the "Healthy and higher" rule and the dwell; a /// pair that only ever reports Healthy levels (250/240) never meets either condition. /// /// public partial class DataConnectionActor { /// The lowest ServiceLevel in the OPC UA Healthy band (Part 4 §6.6.2.4.2). internal const byte HealthyServiceLevelMinimum = 200; /// /// Part 4 §6.6.2.4.2 "Maintenance": the server is not available for use — OtOpcUa reports it while /// it is shutting down or its endpoint has failed. Skips the degraded window (ScadaBridge#141). /// internal const byte MaintenanceServiceLevel = 0; private const string ServiceLevelPollTimerKey = "servicelevel-poll"; private ServiceLevelFailoverPolicy _serviceLevelPolicy; /// When the current unbroken run of Degraded samples began; null when not degraded. private DateTimeOffset? _degradedSince; /// When the last ServiceLevel-driven switch happened (dwell / anti-flap). private DateTimeOffset _lastServiceLevelSwitchAt = DateTimeOffset.MinValue; private bool _serviceLevelReadInFlight; private bool _serviceLevelProbeInFlight; /// True until the first sample of a session is evaluated (connect-time preference). private bool _serviceLevelConnectCheckPending; /// /// The last level this session actually read (unknown samples do not overwrite it); null when /// none has been read. Drives the lost-link probe (ScadaBridge#141). /// private byte? _lastKnownServiceLevel; /// /// True while the lost-link probe of the other endpoint is outstanding; the first reconnect /// attempt is held until it answers (ScadaBridge#141). /// private bool _disconnectProbePending; /// /// True once ServiceLevel 0 has started a probe in the current degraded streak: 0 skips the window /// once, so a peer that is not Healthy is then re-probed once per window, not on every poll. /// private bool _goingAwayProbeStarted; /// When the outstanding lost-link probe started (to shorten the reconnect wait after it). private DateTimeOffset _disconnectProbeStartedAt; /// Why is re-establishing the link. internal enum ReconnectCause { /// The link was lost (disconnect, connection-level failure). ConnectionLost, /// An in-place reconfiguration swapped the adapter. Reconfigured, /// A sustained Degraded ServiceLevel swapped to the other endpoint (#138). ServiceLevelFailover, } /// /// True when this connection's ServiceLevel is watched: the policy is enabled, there is a /// backup to go to, and the live adapter can read a ServiceLevel. /// private bool ServiceLevelFailoverInForce => _serviceLevelPolicy.Enabled && _backupConfig != null && _adapter is IServiceLevelReadable; /// Called on entering Connected. Arms the poll (first tick immediately). private void StartServiceLevelMonitoring() { ResetServiceLevelSession(); if (!ServiceLevelFailoverInForce) return; _serviceLevelConnectCheckPending = true; Timers.StartPeriodicTimer(ServiceLevelPollTimerKey, ServiceLevelPollTick.Instance, TimeSpan.Zero, _options.ServiceLevelPollInterval); } /// Called on leaving Connected. Results still in flight are fenced by generation. private void StopServiceLevelMonitoring() { Timers.Cancel(ServiceLevelPollTimerKey); ResetServiceLevelSession(); } private void ResetServiceLevelSession() { _degradedSince = null; _serviceLevelReadInFlight = false; _serviceLevelProbeInFlight = false; _serviceLevelConnectCheckPending = false; _lastKnownServiceLevel = null; _goingAwayProbeStarted = false; } private void HandleServiceLevelPollTick() { // One read at a time, and none while the other endpoint is being probed: the probe's // verdict is taken against the level it started from. if (!ServiceLevelFailoverInForce || _serviceLevelReadInFlight || _serviceLevelProbeInFlight) return; _serviceLevelReadInFlight = true; var generation = _adapterGeneration; var reader = (IServiceLevelReadable)_adapter; var timeout = _options.ServiceLevelProbeTimeout; Task.Run(() => reader.ReadServiceLevelAsync().WaitAsync(timeout)) .ContinueWith(t => new ServiceLevelSampled(generation, t.IsCompletedSuccessfully ? t.Result : null), TaskScheduler.Default) .PipeTo(_self); } private void HandleServiceLevelSampled(ServiceLevelSampled sampled) { if (sampled.AdapterGeneration != _adapterGeneration) return; _serviceLevelReadInFlight = false; var connectCheck = _serviceLevelConnectCheckPending; _serviceLevelConnectCheckPending = false; if (sampled.Level is { } known) _lastKnownServiceLevel = known; if (sampled.Level is not { } level || level >= HealthyServiceLevelMinimum) { // Healthy, or unknown: either way the run of Degraded samples is broken. Unknown never // counts toward a switch — acting on missing information is how a pair flaps. if (_degradedSince != null) _log.Info("[{0}] {1} endpoint ServiceLevel is {2} — degraded watch cleared", _connectionName, _activeEndpoint, sampled.Level?.ToString(CultureInfo.InvariantCulture) ?? "unknown"); _degradedSince = null; _goingAwayProbeStarted = false; return; } var now = DateTimeOffset.UtcNow; // ScadaBridge#141: 0 means the server is going away (graceful stop) — it will not be there // for a full window, so the other endpoint is checked now. Still "Healthy and higher" + dwell. var goingAway = level == MaintenanceServiceLevel && !_goingAwayProbeStarted; if (_degradedSince is null) { _degradedSince = now; _log.Warning("[{0}] {1} endpoint reports ServiceLevel {2} (Degraded band, < {3}) — " + "will check the other endpoint {4}", _connectionName, _activeEndpoint, level, HealthyServiceLevelMinimum, connectCheck ? "now (connect-time preference)" : goingAway ? "now (ServiceLevel 0: the server is going away)" : $"if it persists for {_serviceLevelPolicy.DegradedWindow.TotalSeconds:F0}s"); } if (!connectCheck && !goingAway && now - _degradedSince.Value < _serviceLevelPolicy.DegradedWindow) return; if (now - _lastServiceLevelSwitchAt < _serviceLevelPolicy.MinDwell) { _log.Debug("[{0}] ServiceLevel {1} still Degraded but the last ServiceLevel switch was {2:F0}s ago " + "(dwell {3:F0}s) — staying", _connectionName, level, (now - _lastServiceLevelSwitchAt).TotalSeconds, _serviceLevelPolicy.MinDwell.TotalSeconds); return; } if (goingAway) _goingAwayProbeStarted = true; StartServiceLevelProbe(level); } /// /// Reads the OTHER endpoint's ServiceLevel through a short-lived adapter built from its /// configuration — connect, read, dispose — off the actor thread, bounded by /// . The live adapter is untouched. /// private void StartServiceLevelProbe(byte currentLevel, bool afterDisconnect = false) { if (!afterDisconnect) _serviceLevelProbeInFlight = true; var generation = _adapterGeneration; var otherConfig = _activeEndpoint == ActiveEndpoint.Primary ? _backupConfig! : _primaryConfig; var factory = _factory; var protocol = _protocolType; var timeout = _options.ServiceLevelProbeTimeout; Task.Run(() => ProbeServiceLevelAsync(factory, protocol, otherConfig).WaitAsync(timeout)) .ContinueWith(t => t.IsCompletedSuccessfully ? new ServiceLevelProbed(generation, currentLevel, t.Result.Level, t.Result.Error, afterDisconnect) : new ServiceLevelProbed(generation, currentLevel, null, t.Exception?.GetBaseException() is TimeoutException ? $"no answer within {timeout.TotalSeconds:F0}s" : t.Exception?.GetBaseException().Message ?? "probe cancelled", afterDisconnect), TaskScheduler.Default) .PipeTo(_self); } private static async Task<(byte? Level, string? Error)> ProbeServiceLevelAsync( IDataConnectionFactory factory, string protocol, IDictionary config) { IDataConnection? probe = null; try { probe = factory.Create(protocol, config); if (probe is not IServiceLevelReadable reader) return (null, "the adapter cannot read a ServiceLevel"); await probe.ConnectAsync(config).ConfigureAwait(false); var level = await reader.ReadServiceLevelAsync().ConfigureAwait(false); return (level, level is null ? "ServiceLevel unreadable" : null); } catch (Exception ex) { return (null, ex.GetBaseException().Message); } finally { // Fire-and-forget: closing a session against a black-holed server can block for the // operation timeout, and the probe's verdict must not wait for it. if (probe != null) _ = probe.DisposeAsync().AsTask().ContinueWith( t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted); } } private void HandleServiceLevelProbed(ServiceLevelProbed probed) { // A lost-link probe belongs to the Reconnecting state (HandleDisconnectProbed). if (probed.AfterDisconnect || probed.AdapterGeneration != _adapterGeneration) return; _serviceLevelProbeInFlight = false; if (!ServiceLevelFailoverInForce || _degradedSince is null) return; var other = _activeEndpoint == ActiveEndpoint.Primary ? ActiveEndpoint.Backup : ActiveEndpoint.Primary; var now = DateTimeOffset.UtcNow; if (probed.OtherLevel is not { } otherLevel) { _log.Warning("[{0}] {1} endpoint ServiceLevel {2} is Degraded but the {3} endpoint is not " + "available ({4}) — staying", _connectionName, _activeEndpoint, probed.CurrentLevel, other, probed.Error); _degradedSince = now; // re-arm: look again after another full window return; } if (otherLevel < HealthyServiceLevelMinimum || otherLevel <= probed.CurrentLevel) { _log.Warning("[{0}] {1} endpoint ServiceLevel {2} is Degraded and the {3} endpoint reports {4} " + "(not a Healthy, higher level) — staying", _connectionName, _activeEndpoint, probed.CurrentLevel, other, otherLevel); _degradedSince = now; return; } var degradedFor = now - _degradedSince.Value; SwitchForServiceLevel(probed.CurrentLevel, otherLevel, $"ServiceLevel {probed.CurrentLevel} Degraded for {degradedFor.TotalSeconds:F0}s"); BecomeReconnecting(ReconnectCause.ServiceLevelFailover); } /// /// The endpoint switch shared by the Connected-state and the lost-link ServiceLevel triggers: /// flips the adapter, resets both connection-failure counters, starts the dwell and logs it. /// The caller re-arms the reconnect. /// private void SwitchForServiceLevel(byte currentLevel, byte otherLevel, string why) { var previousEndpoint = SwitchToOtherEndpoint(); _consecutiveFailures = 0; _consecutiveUnstableDisconnects = 0; _lastServiceLevelSwitchAt = DateTimeOffset.UtcNow; _log.Warning("[{0}] Failing over from {1} to {2} ({3}; {2} reports {4})", _connectionName, previousEndpoint, _activeEndpoint, why, otherLevel); if (_siteEventLogger != null) { _ = _siteEventLogger.LogEventAsync( "connection", "Warning", null, _connectionName, $"Failover from {previousEndpoint} to {_activeEndpoint} (ServiceLevel)", $"{previousEndpoint}: {why} (last ServiceLevel {currentLevel}); {_activeEndpoint} reports {otherLevel}"); } } /// /// ScadaBridge#141 — called by for a LOST link. When the session's /// last known ServiceLevel was Degraded (below 200: the server had said it was not the one to be /// on, typically 0 during a graceful stop), the other endpoint is probed before the first reconnect /// attempt instead of retrying the endpoint that is going away. Not when the policy is not in /// force, the session never read a Degraded level (250/240 pairs, unknown-only), or the dwell of /// the previous ServiceLevel switch has not passed. /// /// The ended session's last known level, captured before it was reset. /// True when the probe was started; the caller must then NOT arm the reconnect timer. private bool TryStartDisconnectProbe(byte? lastKnownLevel) { if (lastKnownLevel is not { } level || level >= HealthyServiceLevelMinimum || !ServiceLevelFailoverInForce) return false; var sinceSwitch = DateTimeOffset.UtcNow - _lastServiceLevelSwitchAt; if (sinceSwitch < _serviceLevelPolicy.MinDwell) { _log.Info("[{0}] Link lost after ServiceLevel {1}, but the last ServiceLevel switch was {2:F0}s ago " + "(dwell {3:F0}s) — normal reconnect cycle", _connectionName, level, sinceSwitch.TotalSeconds, _serviceLevelPolicy.MinDwell.TotalSeconds); return false; } _log.Warning("[{0}] Link to the {1} endpoint lost after it reported ServiceLevel {2} — checking the " + "other endpoint before retrying", _connectionName, _activeEndpoint, level); _disconnectProbePending = true; _disconnectProbeStartedAt = DateTimeOffset.UtcNow; StartServiceLevelProbe(level, afterDisconnect: true); return true; } /// /// ScadaBridge#141 — the lost-link probe's verdict (Reconnecting state). A Healthy level on the /// other endpoint switches to it and reconnects at once; anything else releases the held reconnect /// into the ordinary retry cycle on the same endpoint, minus the time the probe already took. /// private void HandleDisconnectProbed(ServiceLevelProbed probed) { if (!probed.AfterDisconnect || !_disconnectProbePending || probed.AdapterGeneration != _adapterGeneration) return; _disconnectProbePending = false; var other = _activeEndpoint == ActiveEndpoint.Primary ? ActiveEndpoint.Backup : ActiveEndpoint.Primary; if (probed.OtherLevel is { } otherLevel && otherLevel >= HealthyServiceLevelMinimum && otherLevel > probed.CurrentLevel) { SwitchForServiceLevel(probed.CurrentLevel, otherLevel, $"link lost after ServiceLevel {probed.CurrentLevel}"); Timers.StartSingleTimer("reconnect", new AttemptConnect(), TimeSpan.Zero); return; } _log.Warning("[{0}] Link to the {1} endpoint lost after ServiceLevel {2}, but the {3} endpoint {4} — " + "normal reconnect cycle", _connectionName, _activeEndpoint, probed.CurrentLevel, other, probed.OtherLevel is { } l ? $"reports {l} (not a Healthy, higher level)" : $"is not available ({probed.Error})"); var remaining = _options.ReconnectInterval - (DateTimeOffset.UtcNow - _disconnectProbeStartedAt); Timers.StartSingleTimer("reconnect", new AttemptConnect(), remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero); } /// Poll tick for the connected endpoint's ServiceLevel. internal sealed record ServiceLevelPollTick { /// The single instance. public static readonly ServiceLevelPollTick Instance = new(); } /// One ServiceLevel sample of the connected endpoint (null = unknown). internal sealed record ServiceLevelSampled(int AdapterGeneration, byte? Level); /// /// The other endpoint's ServiceLevel (null = unreachable / unreadable, see Error). /// marks the lost-link probe (ScadaBridge#141), which only the /// Reconnecting state acts on. /// internal sealed record ServiceLevelProbed( int AdapterGeneration, byte CurrentLevel, byte? OtherLevel, string? Error, bool AfterDisconnect = false); } /// /// The connection-level ServiceLevel failover settings, read from the PRIMARY endpoint's flattened /// configuration (). Absent keys mean the defaults. /// internal readonly record struct ServiceLevelFailoverPolicy(bool Enabled, TimeSpan DegradedWindow, TimeSpan MinDwell) { /// Parses the policy; malformed or out-of-range values fall back to safe values. /// The primary endpoint's flattened configuration. /// The policy in force. public static ServiceLevelFailoverPolicy From(IDictionary? primaryConfig) { var enabled = true; var window = OpcUaServiceLevelFailoverConfig.DefaultDegradedWindowSeconds; var dwell = OpcUaServiceLevelFailoverConfig.DefaultMinDwellSeconds; if (primaryConfig != null) { if (primaryConfig.TryGetValue(OpcUaEndpointConfigSerializer.ServiceLevelFailoverEnabledKey, out var e) && bool.TryParse(e, out var parsedEnabled)) enabled = parsedEnabled; if (primaryConfig.TryGetValue(OpcUaEndpointConfigSerializer.ServiceLevelFailoverWindowKey, out var w) && int.TryParse(w, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedWindow)) window = Math.Max(1, parsedWindow); if (primaryConfig.TryGetValue(OpcUaEndpointConfigSerializer.ServiceLevelFailoverDwellKey, out var d) && int.TryParse(d, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedDwell)) dwell = Math.Max(0, parsedDwell); } return new ServiceLevelFailoverPolicy(enabled, TimeSpan.FromSeconds(window), TimeSpan.FromSeconds(dwell)); } }