using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; /// /// Coordinates GalaxyDriver's recovery from gateway transport failure. Drives a /// state machine — Healthy → TransportLost → Reopening → Replaying → Healthy /// — and exposes the current state through a snapshot + change event so the /// driver's DriverHealth reflects Degraded while we're not in /// Healthy. /// /// /// /// The supervisor doesn't own the session, the subscription registry, or the /// event pump. It receives transport-failure signals from the rest of the /// driver (EventPump throws, a gw RPC raises, the heartbeat times out), runs /// a one-attempt-at-a-time recovery loop, and lets the rest of the driver /// continue serving cached state during recovery. /// /// /// Reopen: caller-supplied callback that re-opens the gw session + /// re-Registers the MXAccess client. Throws on failure. /// /// /// Replay: caller-supplied callback that re-establishes every active /// subscription. Production wraps gw's ReplaySubscriptionsCommand /// (tracked in the mxaccessgw backlog); when that's not available, the callback falls /// back to walking the SubscriptionRegistry and re-issuing SubscribeBulk for /// every tracked tag. /// /// /// Backoff is capped exponential — first retry after /// , doubled per failed attempt, /// capped at . Persistent failures /// hold the supervisor in Reopening indefinitely; the supervisor never /// gives up on its own — operators / Phase 6.4 soak handle that policy. /// /// public sealed class ReconnectSupervisor : IDisposable { /// Recovery state machine. public enum State { Healthy, TransportLost, Reopening, Replaying, } private readonly Func _reopen; private readonly Func _replay; private readonly ReconnectOptions _options; private readonly ILogger _logger; private readonly Func? _backoffDelay; private readonly Lock _stateLock = new(); private State _state = State.Healthy; private string? _lastError; private DateTime? _lastTransitionUtc; private Task? _recoveryLoop; private CancellationTokenSource? _loopCts; private bool _disposed; private int _cyclesStarted; private int _cyclesReleased; /// Recovery cycles started so far (one per that left Healthy). Test seam. internal int CyclesStarted => Volatile.Read(ref _cyclesStarted); /// Recovery cycles whose has been disposed. Equals /// once every loop has exited — a cycle's CTS used to be released only /// if it happened to be the LAST one when ran. Test seam. internal int CyclesReleased => Volatile.Read(ref _cyclesReleased); /// Fires after every state transition. public event EventHandler? StateChanged; /// /// Initializes a new ReconnectSupervisor with the specified recovery handlers and options. /// /// Callback to reopen the transport connection. /// Callback to replay subscriptions and state after reopening. /// Optional reconnection behavior configuration. /// Optional logger instance; defaults to null logger if not provided. /// Optional custom backoff delay calculator; uses default if not provided. public ReconnectSupervisor( Func reopen, Func replay, ReconnectOptions? options = null, ILogger? logger = null, Func? backoffDelay = null) { _reopen = reopen ?? throw new ArgumentNullException(nameof(reopen)); _replay = replay ?? throw new ArgumentNullException(nameof(replay)); _options = options ?? new ReconnectOptions(); _logger = logger ?? NullLogger.Instance; _backoffDelay = backoffDelay; } /// Current state. Healthy = fully recovered + subscriptions live. public State CurrentState { get { lock (_stateLock) return _state; } } /// True when CurrentState != Healthy. Drivers map this to DriverState.Degraded. public bool IsDegraded { get { lock (_stateLock) return _state != State.Healthy; } } /// Gets the message of the last reported transport error, if any. public string? LastError { get { lock (_stateLock) return _lastError; } } /// Gets the UTC timestamp of the most recent state transition. public DateTime? LastTransitionUtc { get { lock (_stateLock) return _lastTransitionUtc; } } /// /// Notify the supervisor that a gw transport failure has been observed. Idempotent — /// repeated calls during an in-flight recovery do not start a parallel loop. The /// first call spawns a background task that drives reopen → replay until it /// succeeds or cancels it. /// /// The exception that triggered the failure notification. public void ReportTransportFailure(Exception cause) { ArgumentNullException.ThrowIfNull(cause); ObjectDisposedException.ThrowIf(_disposed, this); lock (_stateLock) { _lastError = cause.Message; if (_state != State.Healthy) { // Already recovering — nothing else to do. _logger.LogDebug("Transport failure reported during {State}: {Message}", _state, cause.Message); return; } TransitionLocked(State.TransportLost, cause.Message); // Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's // finally). Before, every cycle allocated a fresh CTS and only the last one was ever // disposed — one leaked CancellationTokenSource per outage for the driver's lifetime. var cts = new CancellationTokenSource(); _loopCts = cts; Interlocked.Increment(ref _cyclesStarted); _recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts)); } } /// /// Wait until the current recovery cycle reaches Healthy or the supplied token /// is cancelled. Returns immediately when already Healthy. Useful for tests /// and for orchestration that wants to gate calls on recovery completing. /// /// Token to cancel the wait operation. /// A task that represents the asynchronous operation. public async Task WaitForHealthyAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested && IsDegraded) { await Task.Delay(50, cancellationToken).ConfigureAwait(false); } } private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts) { try { await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false); } finally { // Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may // race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object. lock (_stateLock) { if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null; } cycleCts.Dispose(); Interlocked.Increment(ref _cyclesReleased); } } private async Task RecoveryLoopCoreAsync(CancellationToken ct) { var attempt = 0; while (!ct.IsCancellationRequested) { attempt++; if (attempt > 1) { var delay = ComputeBackoff(attempt); _logger.LogInformation( "Galaxy reconnect attempt {Attempt} — waiting {Delay} before retry", attempt, delay); try { await Task.Delay(delay, ct).ConfigureAwait(false); } catch (OperationCanceledException) { return; } } // === Reopening phase === lock (_stateLock) TransitionLocked(State.Reopening, _lastError); try { await _reopen(ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy reopen failed (attempt {Attempt})", attempt); lock (_stateLock) { _lastError = ex.Message; } continue; // back to backoff + retry } // === Replaying phase === lock (_stateLock) TransitionLocked(State.Replaying, _lastError); try { await _replay(ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy replay failed (attempt {Attempt})", attempt); lock (_stateLock) { _lastError = ex.Message; } continue; // back to backoff + retry } // === Done === lock (_stateLock) { _lastError = null; TransitionLocked(State.Healthy, null); } _logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt); return; } } private TimeSpan ComputeBackoff(int attempt) { if (_backoffDelay is not null) return _backoffDelay(attempt, _options.InitialBackoff, _options.MaxBackoff); // Standard capped exponential — InitialBackoff * 2^(attempt-2), capped at MaxBackoff. // Attempt 2 → InitialBackoff, attempt 3 → 2x, attempt 4 → 4x, etc. var multiplier = Math.Min(1L << Math.Max(0, attempt - 2), int.MaxValue); var ticks = _options.InitialBackoff.Ticks * multiplier; if (ticks <= 0 || ticks > _options.MaxBackoff.Ticks) ticks = _options.MaxBackoff.Ticks; return TimeSpan.FromTicks(ticks); } private void TransitionLocked(State next, string? cause) { if (next == _state) return; var previous = _state; _state = next; _lastTransitionUtc = DateTime.UtcNow; var transition = new StateTransition(previous, next, cause, _lastTransitionUtc.Value); try { StateChanged?.Invoke(this, transition); } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy reconnect StateChanged handler threw — continuing."); } } /// /// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's /// thread. The async counterpart of ; the driver's ShutdownAsync /// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent. /// /// A task that completes once the recovery loop has exited. public async Task StopAsync() { if (_disposed) return; _disposed = true; CancellationTokenSource? cts; Task? loop; lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; } // The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS // throws on Cancel, and a disposed one is a no-op on Dispose. try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ } if (loop is not null) { try { await loop.ConfigureAwait(false); } catch { /* shutdown */ } } cts?.Dispose(); } /// Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits). public void Dispose() => StopAsync().GetAwaiter().GetResult(); } /// /// One state transition observed by the supervisor. /// public sealed record StateTransition( ReconnectSupervisor.State Previous, ReconnectSupervisor.State Next, string? Cause, DateTime AtUtc); /// /// Knobs for the supervisor's backoff. on the driver /// options record (PR 4.0) maps onto this — they're separate types so the supervisor /// can be exercised in tests without the full driver options surface. /// public sealed record ReconnectOptions( TimeSpan? InitialBackoffOverride = null, TimeSpan? MaxBackoffOverride = null) { /// Gets the initial backoff delay for reconnection attempts. public TimeSpan InitialBackoff => InitialBackoffOverride ?? TimeSpan.FromMilliseconds(500); /// Gets the maximum backoff delay for reconnection attempts. public TimeSpan MaxBackoff => MaxBackoffOverride ?? TimeSpan.FromSeconds(30); }