ScadaBridge/src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/Actors/DataConnectionActor.ServiceLevel.cs

using System.Globalization;
using Akka.Actor;
using Akka.Event;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;

namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;

/// <summary>
/// ScadaBridge#138 — ServiceLevel-aware failover between a primary and a backup endpoint.
///
/// <para>
/// <b>Why.</b> Until now the connection failed over only when the link itself misbehaved
/// (<c>FailoverRetryCount</c> consecutive connect failures or unstable disconnects). Under OtOpcUa
/// Warm redundancy (lmxopcua#717) the standby keeps a perfectly healthy session but serves
/// <c>BadOutOfService</c> data at a standby ServiceLevel (90 healthy / 50 degraded / 1 booting), so a
/// connection that lands there would sit on Bad quality indefinitely.
/// </para>
///
/// <para>
/// <b>What.</b> While Connected, the connected endpoint's <c>Server_ServiceLevel</c> is polled every
/// <see cref="DataConnectionOptions.ServiceLevelPollInterval"/>. A level below
/// <see cref="HealthyServiceLevelMinimum"/> (200, the bottom of the Part 4 Healthy band) starts a
/// degraded streak; any Healthy or unreadable sample ends it. Once the streak has lasted the
/// connection's degraded window — or immediately on the first sample after a (re)connect, which is
/// the "prefer the better endpoint on connect" rule — the other endpoint is probed with a
/// short-lived session. The connection switches only when that probe reads a Healthy level that is
/// also higher than the connected one, and never within the dwell time of the previous
/// ServiceLevel switch. Otherwise it stays put and the window re-arms.
/// </para>
///
/// <para>
/// <b>Unchanged behaviour.</b> A level of 200 or more never starts a streak, so today's OtOpcUa pair
/// (250 Primary / 240 Secondary) never triggers anything; an adapter without
/// <see cref="IServiceLevelReadable"/> (MxGateway) and a connection without a backup are never
/// monitored at all. The switch itself reuses <see cref="SwitchToOtherEndpoint"/> and the reconnect
/// cycle, exactly like the connection-driven failovers.
/// </para>
///
/// <para>
/// <b>A server going away (ScadaBridge#141).</b> A graceful stop of a redundant OtOpcUa node reports
/// ServiceLevel 0 for only a few seconds before the session closes — shorter than the degraded window
/// — and the following unreadable sample cleared the streak, so the connection fell back to the
/// connectivity path and retried the stopped node <c>FailoverRetryCount</c> times (66 s on QA). Two
/// going-away triggers now skip that wait: (1) a sample of <see cref="MaintenanceServiceLevel"/> (0,
/// Part 4 "Maintenance": the server is not available for use) probes the other endpoint at once,
/// without the window; and (2) when the link is lost while the last KNOWN level of the session was
/// Degraded (below 200), the other endpoint is probed before the first reconnect attempt and, if it
/// answers Healthy, the connection switches straight to it instead of spending the retry count on the
/// endpoint that announced it was degraded. Both keep the "Healthy and higher" rule and the dwell; a
/// pair that only ever reports Healthy levels (250/240) never meets either condition.
/// </para>
/// </summary>
public partial class DataConnectionActor
{
    /// <summary>The lowest ServiceLevel in the OPC UA Healthy band (Part 4 §6.6.2.4.2).</summary>
    internal const byte HealthyServiceLevelMinimum = 200;

    /// <summary>
    /// Part 4 §6.6.2.4.2 "Maintenance": the server is not available for use — OtOpcUa reports it while
    /// it is shutting down or its endpoint has failed. Skips the degraded window (ScadaBridge#141).
    /// </summary>
    internal const byte MaintenanceServiceLevel = 0;

    private const string ServiceLevelPollTimerKey = "servicelevel-poll";

    private ServiceLevelFailoverPolicy _serviceLevelPolicy;

    /// <summary>When the current unbroken run of Degraded samples began; null when not degraded.</summary>
    private DateTimeOffset? _degradedSince;

    /// <summary>When the last ServiceLevel-driven switch happened (dwell / anti-flap).</summary>
    private DateTimeOffset _lastServiceLevelSwitchAt = DateTimeOffset.MinValue;

    private bool _serviceLevelReadInFlight;
    private bool _serviceLevelProbeInFlight;

    /// <summary>True until the first sample of a session is evaluated (connect-time preference).</summary>
    private bool _serviceLevelConnectCheckPending;

    /// <summary>
    /// The last level this session actually read (unknown samples do not overwrite it); null when
    /// none has been read. Drives the lost-link probe (ScadaBridge#141).
    /// </summary>
    private byte? _lastKnownServiceLevel;

    /// <summary>
    /// True while the lost-link probe of the other endpoint is outstanding; the first reconnect
    /// attempt is held until it answers (ScadaBridge#141).
    /// </summary>
    private bool _disconnectProbePending;

    /// <summary>
    /// True once ServiceLevel 0 has started a probe in the current degraded streak: 0 skips the window
    /// once, so a peer that is not Healthy is then re-probed once per window, not on every poll.
    /// </summary>
    private bool _goingAwayProbeStarted;

    /// <summary>When the outstanding lost-link probe started (to shorten the reconnect wait after it).</summary>
    private DateTimeOffset _disconnectProbeStartedAt;

    /// <summary>Why <see cref="BecomeReconnecting"/> is re-establishing the link.</summary>
    internal enum ReconnectCause
    {
        /// <summary>The link was lost (disconnect, connection-level failure).</summary>
        ConnectionLost,

        /// <summary>An in-place reconfiguration swapped the adapter.</summary>
        Reconfigured,

        /// <summary>A sustained Degraded ServiceLevel swapped to the other endpoint (#138).</summary>
        ServiceLevelFailover,
    }

    /// <summary>
    /// True when this connection's ServiceLevel is watched: the policy is enabled, there is a
    /// backup to go to, and the live adapter can read a ServiceLevel.
    /// </summary>
    private bool ServiceLevelFailoverInForce =>
        _serviceLevelPolicy.Enabled && _backupConfig != null && _adapter is IServiceLevelReadable;

    /// <summary>Called on entering Connected. Arms the poll (first tick immediately).</summary>
    private void StartServiceLevelMonitoring()
    {
        ResetServiceLevelSession();
        if (!ServiceLevelFailoverInForce)
            return;

        _serviceLevelConnectCheckPending = true;
        Timers.StartPeriodicTimer(ServiceLevelPollTimerKey, ServiceLevelPollTick.Instance,
            TimeSpan.Zero, _options.ServiceLevelPollInterval);
    }

    /// <summary>Called on leaving Connected. Results still in flight are fenced by generation.</summary>
    private void StopServiceLevelMonitoring()
    {
        Timers.Cancel(ServiceLevelPollTimerKey);
        ResetServiceLevelSession();
    }

    private void ResetServiceLevelSession()
    {
        _degradedSince = null;
        _serviceLevelReadInFlight = false;
        _serviceLevelProbeInFlight = false;
        _serviceLevelConnectCheckPending = false;
        _lastKnownServiceLevel = null;
        _goingAwayProbeStarted = false;
    }

    private void HandleServiceLevelPollTick()
    {
        // One read at a time, and none while the other endpoint is being probed: the probe's
        // verdict is taken against the level it started from.
        if (!ServiceLevelFailoverInForce || _serviceLevelReadInFlight || _serviceLevelProbeInFlight)
            return;

        _serviceLevelReadInFlight = true;
        var generation = _adapterGeneration;
        var reader = (IServiceLevelReadable)_adapter;
        var timeout = _options.ServiceLevelProbeTimeout;
        Task.Run(() => reader.ReadServiceLevelAsync().WaitAsync(timeout))
            .ContinueWith(t => new ServiceLevelSampled(generation,
                t.IsCompletedSuccessfully ? t.Result : null), TaskScheduler.Default)
            .PipeTo(_self);
    }

    private void HandleServiceLevelSampled(ServiceLevelSampled sampled)
    {
        if (sampled.AdapterGeneration != _adapterGeneration)
            return;

        _serviceLevelReadInFlight = false;
        var connectCheck = _serviceLevelConnectCheckPending;
        _serviceLevelConnectCheckPending = false;
        if (sampled.Level is { } known)
            _lastKnownServiceLevel = known;

        if (sampled.Level is not { } level || level >= HealthyServiceLevelMinimum)
        {
            // Healthy, or unknown: either way the run of Degraded samples is broken. Unknown never
            // counts toward a switch — acting on missing information is how a pair flaps.
            if (_degradedSince != null)
                _log.Info("[{0}] {1} endpoint ServiceLevel is {2} — degraded watch cleared",
                    _connectionName, _activeEndpoint, sampled.Level?.ToString(CultureInfo.InvariantCulture) ?? "unknown");
            _degradedSince = null;
            _goingAwayProbeStarted = false;
            return;
        }

        var now = DateTimeOffset.UtcNow;
        // ScadaBridge#141: 0 means the server is going away (graceful stop) — it will not be there
        // for a full window, so the other endpoint is checked now. Still "Healthy and higher" + dwell.
        var goingAway = level == MaintenanceServiceLevel && !_goingAwayProbeStarted;
        if (_degradedSince is null)
        {
            _degradedSince = now;
            _log.Warning("[{0}] {1} endpoint reports ServiceLevel {2} (Degraded band, < {3}) — "
                + "will check the other endpoint {4}",
                _connectionName, _activeEndpoint, level, HealthyServiceLevelMinimum,
                connectCheck ? "now (connect-time preference)"
                    : goingAway ? "now (ServiceLevel 0: the server is going away)"
                    : $"if it persists for {_serviceLevelPolicy.DegradedWindow.TotalSeconds:F0}s");
        }

        if (!connectCheck && !goingAway && now - _degradedSince.Value < _serviceLevelPolicy.DegradedWindow)
            return;

        if (now - _lastServiceLevelSwitchAt < _serviceLevelPolicy.MinDwell)
        {
            _log.Debug("[{0}] ServiceLevel {1} still Degraded but the last ServiceLevel switch was {2:F0}s ago "
                + "(dwell {3:F0}s) — staying", _connectionName, level,
                (now - _lastServiceLevelSwitchAt).TotalSeconds, _serviceLevelPolicy.MinDwell.TotalSeconds);
            return;
        }

        if (goingAway)
            _goingAwayProbeStarted = true;
        StartServiceLevelProbe(level);
    }

    /// <summary>
    /// Reads the OTHER endpoint's ServiceLevel through a short-lived adapter built from its
    /// configuration — connect, read, dispose — off the actor thread, bounded by
    /// <see cref="DataConnectionOptions.ServiceLevelProbeTimeout"/>. The live adapter is untouched.
    /// </summary>
    private void StartServiceLevelProbe(byte currentLevel, bool afterDisconnect = false)
    {
        if (!afterDisconnect)
            _serviceLevelProbeInFlight = true;
        var generation = _adapterGeneration;
        var otherConfig = _activeEndpoint == ActiveEndpoint.Primary ? _backupConfig! : _primaryConfig;
        var factory = _factory;
        var protocol = _protocolType;
        var timeout = _options.ServiceLevelProbeTimeout;

        Task.Run(() => ProbeServiceLevelAsync(factory, protocol, otherConfig).WaitAsync(timeout))
            .ContinueWith(t => t.IsCompletedSuccessfully
                    ? new ServiceLevelProbed(generation, currentLevel, t.Result.Level, t.Result.Error, afterDisconnect)
                    : new ServiceLevelProbed(generation, currentLevel, null,
                        t.Exception?.GetBaseException() is TimeoutException
                            ? $"no answer within {timeout.TotalSeconds:F0}s"
                            : t.Exception?.GetBaseException().Message ?? "probe cancelled",
                        afterDisconnect),
                TaskScheduler.Default)
            .PipeTo(_self);
    }

    private static async Task<(byte? Level, string? Error)> ProbeServiceLevelAsync(
        IDataConnectionFactory factory, string protocol, IDictionary<string, string> config)
    {
        IDataConnection? probe = null;
        try
        {
            probe = factory.Create(protocol, config);
            if (probe is not IServiceLevelReadable reader)
                return (null, "the adapter cannot read a ServiceLevel");

            await probe.ConnectAsync(config).ConfigureAwait(false);
            var level = await reader.ReadServiceLevelAsync().ConfigureAwait(false);
            return (level, level is null ? "ServiceLevel unreadable" : null);
        }
        catch (Exception ex)
        {
            return (null, ex.GetBaseException().Message);
        }
        finally
        {
            // Fire-and-forget: closing a session against a black-holed server can block for the
            // operation timeout, and the probe's verdict must not wait for it.
            if (probe != null)
                _ = probe.DisposeAsync().AsTask().ContinueWith(
                    t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted);
        }
    }

    private void HandleServiceLevelProbed(ServiceLevelProbed probed)
    {
        // A lost-link probe belongs to the Reconnecting state (HandleDisconnectProbed).
        if (probed.AfterDisconnect || probed.AdapterGeneration != _adapterGeneration)
            return;

        _serviceLevelProbeInFlight = false;
        if (!ServiceLevelFailoverInForce || _degradedSince is null)
            return;

        var other = _activeEndpoint == ActiveEndpoint.Primary ? ActiveEndpoint.Backup : ActiveEndpoint.Primary;
        var now = DateTimeOffset.UtcNow;

        if (probed.OtherLevel is not { } otherLevel)
        {
            _log.Warning("[{0}] {1} endpoint ServiceLevel {2} is Degraded but the {3} endpoint is not "
                + "available ({4}) — staying", _connectionName, _activeEndpoint, probed.CurrentLevel, other,
                probed.Error);
            _degradedSince = now; // re-arm: look again after another full window
            return;
        }

        if (otherLevel < HealthyServiceLevelMinimum || otherLevel <= probed.CurrentLevel)
        {
            _log.Warning("[{0}] {1} endpoint ServiceLevel {2} is Degraded and the {3} endpoint reports {4} "
                + "(not a Healthy, higher level) — staying", _connectionName, _activeEndpoint,
                probed.CurrentLevel, other, otherLevel);
            _degradedSince = now;
            return;
        }

        var degradedFor = now - _degradedSince.Value;
        SwitchForServiceLevel(probed.CurrentLevel, otherLevel,
            $"ServiceLevel {probed.CurrentLevel} Degraded for {degradedFor.TotalSeconds:F0}s");
        BecomeReconnecting(ReconnectCause.ServiceLevelFailover);
    }

    /// <summary>
    /// The endpoint switch shared by the Connected-state and the lost-link ServiceLevel triggers:
    /// flips the adapter, resets both connection-failure counters, starts the dwell and logs it.
    /// The caller re-arms the reconnect.
    /// </summary>
    private void SwitchForServiceLevel(byte currentLevel, byte otherLevel, string why)
    {
        var previousEndpoint = SwitchToOtherEndpoint();
        _consecutiveFailures = 0;
        _consecutiveUnstableDisconnects = 0;
        _lastServiceLevelSwitchAt = DateTimeOffset.UtcNow;

        _log.Warning("[{0}] Failing over from {1} to {2} ({3}; {2} reports {4})",
            _connectionName, previousEndpoint, _activeEndpoint, why, otherLevel);

        if (_siteEventLogger != null)
        {
            _ = _siteEventLogger.LogEventAsync(
                "connection", "Warning", null, _connectionName,
                $"Failover from {previousEndpoint} to {_activeEndpoint} (ServiceLevel)",
                $"{previousEndpoint}: {why} (last ServiceLevel {currentLevel}); {_activeEndpoint} reports {otherLevel}");
        }
    }

    /// <summary>
    /// ScadaBridge#141 — called by <see cref="BecomeReconnecting"/> for a LOST link. When the session's
    /// last known ServiceLevel was Degraded (below 200: the server had said it was not the one to be
    /// on, typically 0 during a graceful stop), the other endpoint is probed before the first reconnect
    /// attempt instead of retrying the endpoint that is going away. Not when the policy is not in
    /// force, the session never read a Degraded level (250/240 pairs, unknown-only), or the dwell of
    /// the previous ServiceLevel switch has not passed.
    /// </summary>
    /// <param name="lastKnownLevel">The ended session's last known level, captured before it was reset.</param>
    /// <returns>True when the probe was started; the caller must then NOT arm the reconnect timer.</returns>
    private bool TryStartDisconnectProbe(byte? lastKnownLevel)
    {
        if (lastKnownLevel is not { } level || level >= HealthyServiceLevelMinimum || !ServiceLevelFailoverInForce)
            return false;

        var sinceSwitch = DateTimeOffset.UtcNow - _lastServiceLevelSwitchAt;
        if (sinceSwitch < _serviceLevelPolicy.MinDwell)
        {
            _log.Info("[{0}] Link lost after ServiceLevel {1}, but the last ServiceLevel switch was {2:F0}s ago "
                + "(dwell {3:F0}s) — normal reconnect cycle", _connectionName, level,
                sinceSwitch.TotalSeconds, _serviceLevelPolicy.MinDwell.TotalSeconds);
            return false;
        }

        _log.Warning("[{0}] Link to the {1} endpoint lost after it reported ServiceLevel {2} — checking the "
            + "other endpoint before retrying", _connectionName, _activeEndpoint, level);
        _disconnectProbePending = true;
        _disconnectProbeStartedAt = DateTimeOffset.UtcNow;
        StartServiceLevelProbe(level, afterDisconnect: true);
        return true;
    }

    /// <summary>
    /// ScadaBridge#141 — the lost-link probe's verdict (Reconnecting state). A Healthy level on the
    /// other endpoint switches to it and reconnects at once; anything else releases the held reconnect
    /// into the ordinary retry cycle on the same endpoint, minus the time the probe already took.
    /// </summary>
    private void HandleDisconnectProbed(ServiceLevelProbed probed)
    {
        if (!probed.AfterDisconnect || !_disconnectProbePending || probed.AdapterGeneration != _adapterGeneration)
            return;

        _disconnectProbePending = false;
        var other = _activeEndpoint == ActiveEndpoint.Primary ? ActiveEndpoint.Backup : ActiveEndpoint.Primary;

        if (probed.OtherLevel is { } otherLevel
            && otherLevel >= HealthyServiceLevelMinimum && otherLevel > probed.CurrentLevel)
        {
            SwitchForServiceLevel(probed.CurrentLevel, otherLevel,
                $"link lost after ServiceLevel {probed.CurrentLevel}");
            Timers.StartSingleTimer("reconnect", new AttemptConnect(), TimeSpan.Zero);
            return;
        }

        _log.Warning("[{0}] Link to the {1} endpoint lost after ServiceLevel {2}, but the {3} endpoint {4} — "
            + "normal reconnect cycle", _connectionName, _activeEndpoint, probed.CurrentLevel, other,
            probed.OtherLevel is { } l ? $"reports {l} (not a Healthy, higher level)" : $"is not available ({probed.Error})");
        var remaining = _options.ReconnectInterval - (DateTimeOffset.UtcNow - _disconnectProbeStartedAt);
        Timers.StartSingleTimer("reconnect", new AttemptConnect(),
            remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero);
    }

    /// <summary>Poll tick for the connected endpoint's ServiceLevel.</summary>
    internal sealed record ServiceLevelPollTick
    {
        /// <summary>The single instance.</summary>
        public static readonly ServiceLevelPollTick Instance = new();
    }

    /// <summary>One ServiceLevel sample of the connected endpoint (null = unknown).</summary>
    internal sealed record ServiceLevelSampled(int AdapterGeneration, byte? Level);

    /// <summary>
    /// The other endpoint's ServiceLevel (null = unreachable / unreadable, see Error).
    /// <paramref name="AfterDisconnect"/> marks the lost-link probe (ScadaBridge#141), which only the
    /// Reconnecting state acts on.
    /// </summary>
    internal sealed record ServiceLevelProbed(
        int AdapterGeneration, byte CurrentLevel, byte? OtherLevel, string? Error, bool AfterDisconnect = false);
}

/// <summary>
/// The connection-level ServiceLevel failover settings, read from the PRIMARY endpoint's flattened
/// configuration (<see cref="OpcUaServiceLevelFailoverConfig"/>). Absent keys mean the defaults.
/// </summary>
internal readonly record struct ServiceLevelFailoverPolicy(bool Enabled, TimeSpan DegradedWindow, TimeSpan MinDwell)
{
    /// <summary>Parses the policy; malformed or out-of-range values fall back to safe values.</summary>
    /// <param name="primaryConfig">The primary endpoint's flattened configuration.</param>
    /// <returns>The policy in force.</returns>
    public static ServiceLevelFailoverPolicy From(IDictionary<string, string>? primaryConfig)
    {
        var enabled = true;
        var window = OpcUaServiceLevelFailoverConfig.DefaultDegradedWindowSeconds;
        var dwell = OpcUaServiceLevelFailoverConfig.DefaultMinDwellSeconds;

        if (primaryConfig != null)
        {
            if (primaryConfig.TryGetValue(OpcUaEndpointConfigSerializer.ServiceLevelFailoverEnabledKey, out var e)
                && bool.TryParse(e, out var parsedEnabled))
                enabled = parsedEnabled;
            if (primaryConfig.TryGetValue(OpcUaEndpointConfigSerializer.ServiceLevelFailoverWindowKey, out var w)
                && int.TryParse(w, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedWindow))
                window = Math.Max(1, parsedWindow);
            if (primaryConfig.TryGetValue(OpcUaEndpointConfigSerializer.ServiceLevelFailoverDwellKey, out var d)
                && int.TryParse(d, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedDwell))
                dwell = Math.Max(0, parsedDwell);
        }

        return new ServiceLevelFailoverPolicy(enabled, TimeSpan.FromSeconds(window), TimeSpan.FromSeconds(dwell));
    }
}