using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
///
/// Coordinates GalaxyDriver's recovery from gateway transport failure. Drives a
/// state machine — Healthy → TransportLost → Reopening → Replaying → Healthy
/// — and exposes the current state through a snapshot + change event so the
/// driver's DriverHealth reflects Degraded while we're not in
/// Healthy.
///
///
///
/// The supervisor doesn't own the session, the subscription registry, or the
/// event pump. It receives transport-failure signals from the rest of the
/// driver (EventPump throws, a gw RPC raises, the heartbeat times out), runs
/// a one-attempt-at-a-time recovery loop, and lets the rest of the driver
/// continue serving cached state during recovery.
///
///
/// Reopen: caller-supplied callback that re-opens the gw session +
/// re-Registers the MXAccess client. Throws on failure.
///
///
/// Replay: caller-supplied callback that re-establishes every active
/// subscription. Production wraps gw's ReplaySubscriptionsCommand
/// (tracked in the mxaccessgw backlog); when that's not available, the callback falls
/// back to walking the SubscriptionRegistry and re-issuing SubscribeBulk for
/// every tracked tag.
///
///
/// Backoff is capped exponential — first retry after
/// , doubled per failed attempt,
/// capped at . Persistent failures
/// hold the supervisor in Reopening indefinitely; the supervisor never
/// gives up on its own — operators / Phase 6.4 soak handle that policy.
///
///
public sealed class ReconnectSupervisor : IDisposable
{
/// Recovery state machine.
public enum State
{
Healthy,
TransportLost,
Reopening,
Replaying,
}
private readonly Func _reopen;
private readonly Func _replay;
private readonly ReconnectOptions _options;
private readonly ILogger _logger;
private readonly Func? _backoffDelay;
private readonly Lock _stateLock = new();
private State _state = State.Healthy;
private string? _lastError;
private DateTime? _lastTransitionUtc;
private Task? _recoveryLoop;
private CancellationTokenSource? _loopCts;
private bool _disposed;
private int _cyclesStarted;
private int _cyclesReleased;
/// Recovery cycles started so far (one per that left Healthy). Test seam.
internal int CyclesStarted => Volatile.Read(ref _cyclesStarted);
/// Recovery cycles whose has been disposed. Equals
/// once every loop has exited — a cycle's CTS used to be released only
/// if it happened to be the LAST one when ran. Test seam.
internal int CyclesReleased => Volatile.Read(ref _cyclesReleased);
/// Fires after every state transition.
public event EventHandler? StateChanged;
///
/// Initializes a new ReconnectSupervisor with the specified recovery handlers and options.
///
/// Callback to reopen the transport connection.
/// Callback to replay subscriptions and state after reopening.
/// Optional reconnection behavior configuration.
/// Optional logger instance; defaults to null logger if not provided.
/// Optional custom backoff delay calculator; uses default if not provided.
public ReconnectSupervisor(
Func reopen,
Func replay,
ReconnectOptions? options = null,
ILogger? logger = null,
Func? backoffDelay = null)
{
_reopen = reopen ?? throw new ArgumentNullException(nameof(reopen));
_replay = replay ?? throw new ArgumentNullException(nameof(replay));
_options = options ?? new ReconnectOptions();
_logger = logger ?? NullLogger.Instance;
_backoffDelay = backoffDelay;
}
/// Current state. Healthy = fully recovered + subscriptions live.
public State CurrentState
{
get { lock (_stateLock) return _state; }
}
/// True when CurrentState != Healthy. Drivers map this to DriverState.Degraded.
public bool IsDegraded
{
get { lock (_stateLock) return _state != State.Healthy; }
}
/// Gets the message of the last reported transport error, if any.
public string? LastError
{
get { lock (_stateLock) return _lastError; }
}
/// Gets the UTC timestamp of the most recent state transition.
public DateTime? LastTransitionUtc
{
get { lock (_stateLock) return _lastTransitionUtc; }
}
///
/// Notify the supervisor that a gw transport failure has been observed. Idempotent —
/// repeated calls during an in-flight recovery do not start a parallel loop. The
/// first call spawns a background task that drives reopen → replay until it
/// succeeds or cancels it.
///
/// The exception that triggered the failure notification.
public void ReportTransportFailure(Exception cause)
{
ArgumentNullException.ThrowIfNull(cause);
ObjectDisposedException.ThrowIf(_disposed, this);
lock (_stateLock)
{
_lastError = cause.Message;
if (_state != State.Healthy)
{
// Already recovering — nothing else to do.
_logger.LogDebug("Transport failure reported during {State}: {Message}", _state, cause.Message);
return;
}
TransitionLocked(State.TransportLost, cause.Message);
// Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's
// finally). Before, every cycle allocated a fresh CTS and only the last one was ever
// disposed — one leaked CancellationTokenSource per outage for the driver's lifetime.
var cts = new CancellationTokenSource();
_loopCts = cts;
Interlocked.Increment(ref _cyclesStarted);
_recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts));
}
}
///
/// Wait until the current recovery cycle reaches Healthy or the supplied token
/// is cancelled. Returns immediately when already Healthy. Useful for tests
/// and for orchestration that wants to gate calls on recovery completing.
///
/// Token to cancel the wait operation.
/// A task that represents the asynchronous operation.
public async Task WaitForHealthyAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested && IsDegraded)
{
await Task.Delay(50, cancellationToken).ConfigureAwait(false);
}
}
private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts)
{
try
{
await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false);
}
finally
{
// Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may
// race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object.
lock (_stateLock)
{
if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null;
}
cycleCts.Dispose();
Interlocked.Increment(ref _cyclesReleased);
}
}
private async Task RecoveryLoopCoreAsync(CancellationToken ct)
{
var attempt = 0;
while (!ct.IsCancellationRequested)
{
attempt++;
if (attempt > 1)
{
var delay = ComputeBackoff(attempt);
_logger.LogInformation(
"Galaxy reconnect attempt {Attempt} — waiting {Delay} before retry", attempt, delay);
try { await Task.Delay(delay, ct).ConfigureAwait(false); }
catch (OperationCanceledException) { return; }
}
// === Reopening phase ===
lock (_stateLock) TransitionLocked(State.Reopening, _lastError);
try
{
await _reopen(ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
catch (Exception ex)
{
_logger.LogWarning(ex, "Galaxy reopen failed (attempt {Attempt})", attempt);
lock (_stateLock) { _lastError = ex.Message; }
continue; // back to backoff + retry
}
// === Replaying phase ===
lock (_stateLock) TransitionLocked(State.Replaying, _lastError);
try
{
await _replay(ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
catch (Exception ex)
{
_logger.LogWarning(ex, "Galaxy replay failed (attempt {Attempt})", attempt);
lock (_stateLock) { _lastError = ex.Message; }
continue; // back to backoff + retry
}
// === Done ===
lock (_stateLock)
{
_lastError = null;
TransitionLocked(State.Healthy, null);
}
_logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt);
return;
}
}
private TimeSpan ComputeBackoff(int attempt)
{
if (_backoffDelay is not null)
return _backoffDelay(attempt, _options.InitialBackoff, _options.MaxBackoff);
// Standard capped exponential — InitialBackoff * 2^(attempt-2), capped at MaxBackoff.
// Attempt 2 → InitialBackoff, attempt 3 → 2x, attempt 4 → 4x, etc.
var multiplier = Math.Min(1L << Math.Max(0, attempt - 2), int.MaxValue);
var ticks = _options.InitialBackoff.Ticks * multiplier;
if (ticks <= 0 || ticks > _options.MaxBackoff.Ticks) ticks = _options.MaxBackoff.Ticks;
return TimeSpan.FromTicks(ticks);
}
private void TransitionLocked(State next, string? cause)
{
if (next == _state) return;
var previous = _state;
_state = next;
_lastTransitionUtc = DateTime.UtcNow;
var transition = new StateTransition(previous, next, cause, _lastTransitionUtc.Value);
try { StateChanged?.Invoke(this, transition); }
catch (Exception ex)
{
_logger.LogWarning(ex,
"Galaxy reconnect StateChanged handler threw — continuing.");
}
}
///
/// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's
/// thread. The async counterpart of ; the driver's ShutdownAsync
/// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent.
///
/// A task that completes once the recovery loop has exited.
public async Task StopAsync()
{
if (_disposed) return;
_disposed = true;
CancellationTokenSource? cts;
Task? loop;
lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }
// The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
// throws on Cancel, and a disposed one is a no-op on Dispose.
try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ }
if (loop is not null)
{
try { await loop.ConfigureAwait(false); } catch { /* shutdown */ }
}
cts?.Dispose();
}
/// Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits).
public void Dispose() => StopAsync().GetAwaiter().GetResult();
}
///
/// One state transition observed by the supervisor.
///
public sealed record StateTransition(
ReconnectSupervisor.State Previous,
ReconnectSupervisor.State Next,
string? Cause,
DateTime AtUtc);
///
/// Knobs for the supervisor's backoff. on the driver
/// options record (PR 4.0) maps onto this — they're separate types so the supervisor
/// can be exercised in tests without the full driver options surface.
///
public sealed record ReconnectOptions(
TimeSpan? InitialBackoffOverride = null,
TimeSpan? MaxBackoffOverride = null)
{
/// Gets the initial backoff delay for reconnection attempts.
public TimeSpan InitialBackoff => InitialBackoffOverride ?? TimeSpan.FromMilliseconds(500);
/// Gets the maximum backoff delay for reconnection attempts.
public TimeSpan MaxBackoff => MaxBackoffOverride ?? TimeSpan.FromSeconds(30);
}