fix(galaxy): make ShutdownAsync a real runtime teardown so re-init rebuilds

GalaxyDriver.ShutdownAsync was a no-op: the MX session, both owned gRPC channels, the EventPump, the alarm feed, the DeployWatcher, the probe watcher and the reconnect supervisor were released only by DisposeAsync, which the host never calls — one leaked gateway session per respawn, removal and /raw browse, and a watchdog Shutdown→Initialize that neither tore down nor rebuilt anything (review 2026-10-21 G-1, G-2, G-9, G-3). - ShutdownAsync and DisposeAsync share one ordered TeardownRuntimeAsync (supervisor first, then deploy/probe watchers, pump, alarm feed, session, channels); owned seams are nulled so the next InitializeAsync rebuilds; injected seams survive a shutdown and drop only on the final dispose. Lifecycle calls are serialised behind a gate. - A failed production connect releases its partial runtime (no orphaned client per watchdog retry) and lands Faulted before and after. - StartDeployWatcherAsync stops the previous watcher before starting a replacement, and now starts on a production instance's FIRST InitializeAsync (the old guard returned whenever discovery had not run here, which on a runtime instance is always). - ReconnectSupervisor releases each cycle's CancellationTokenSource when its loop exits and gains an awaitable StopAsync; PerPlatformProbeWatcher gains an awaitable StopAsync. - EventPump treats any exception under cancellation as a clean stop, so a teardown never wakes the supervisor. - Internal seams for tests: an IGalaxyDeployWatchSource ctor parameter and a session-open override forwarded to the owned GalaxyMxSession. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 06:30:14 -04:00
parent ad2b5c4044
commit 2d25c916b6
7 changed files with 879 additions and 102 deletions
@@ -146,9 +146,61 @@ public sealed class GalaxyDriver
    private HostConnectivityForwarder? _transportForwarder;
    private PerPlatformProbeWatcher? _probeWatcher;

+    // Injected deploy-watch seam (tests). Null in production, where StartDeployWatcherAsync builds a
+    // GatewayGalaxyDeployWatchSource over the owned GalaxyRepositoryClient.
+    private readonly IGalaxyDeployWatchSource? _deployWatchSource;
+
+    // True while the seams (_subscriber / _dataWriter / _hierarchySource / _alarmFeed /
+    // _alarmAcknowledger) were built by BuildProductionRuntimeAsync and are therefore ours to dispose
+    // and null on ShutdownAsync — the host's comms-loss recovery is ShutdownAsync → InitializeAsync on
+    // this SAME object, and InitializeAsync rebuilds only when the seams are null. Injected seams
+    // belong to the injector: ShutdownAsync stops every loop it started on top of them but leaves the
+    // seams in place, so a seam-driven test can cycle the lifecycle the way the watchdog does.
+    private bool _ownsRuntime;
+
+    // Serialises InitializeAsync / ShutdownAsync / DisposeAsync so a late watchdog re-init can never
+    // interleave with a teardown and resurrect half of the runtime it is tearing down.
+    private readonly SemaphoreSlim _lifecycleGate = new(1, 1);
+
+    // Set for the duration of a runtime teardown so the pump's stream-fault / replay-gap callbacks and
+    // the supervisor's replay cannot start a fresh pump underneath it.
+    private volatile bool _runtimeStopping;
+
    private DriverHealth _health = new(DriverState.Unknown, null, null);
    private bool _disposed;

+    /// <summary>
+    ///     Test seam — forwarded to <see cref="GalaxyMxSession.OpenAndRegisterOverrideForTests"/> on the
+    ///     session <see cref="BuildProductionRuntimeAsync"/> creates, so the owned-runtime lifecycle
+    ///     (client + session construction, teardown, rebuild) can be exercised without a gateway. The
+    ///     gRPC channels it builds are real but lazy: nothing dials until a call is made.
+    /// </summary>
+    internal Func<CancellationToken, Task>? SessionOpenOverrideForTests { get; set; }
+
+    /// <summary>Test seam: the owned gateway client, null when no production runtime is live.</summary>
+    internal MxGatewayClient? OwnedMxClientForTests => _ownedMxClient;
+
+    /// <summary>Test seam: the owned MX session, null when no production runtime is live.</summary>
+    internal GalaxyMxSession? OwnedMxSessionForTests => _ownedMxSession;
+
+    /// <summary>Test seam: the owned repository client, null until the deploy watcher or discovery built it.</summary>
+    internal GalaxyRepositoryClient? OwnedRepositoryClientForTests => _ownedRepositoryClient;
+
+    /// <summary>Test seam: the live reconnect supervisor, null outside a production runtime.</summary>
+    internal ReconnectSupervisor? SupervisorForTests => _supervisor;
+
+    /// <summary>Test seam: the live per-platform probe watcher, null until InitializeAsync.</summary>
+    internal PerPlatformProbeWatcher? ProbeWatcherForTests => _probeWatcher;
+
+    /// <summary>Test seam: the live deploy watcher, null until InitializeAsync starts one.</summary>
+    internal DeployWatcher? DeployWatcherForTests => _deployWatcher;
+
+    /// <summary>Test seam: true while an <see cref="EventPump"/> is running.</summary>
+    internal bool HasEventPumpForTests { get { lock (_pumpLock) return _eventPump is not null; } }
+
+    /// <summary>Test seam: the number of driver subscriptions the registry still tracks.</summary>
+    internal int TrackedSubscriptionCountForTests => _subscriptions.TrackedSubscriptionCount;
+
    /// <summary>
    ///     Server-pushed data-change notification. Fires from the
    ///     <see cref="EventPump"/>'s background loop; handlers should be cheap (or queue
@@ -202,6 +254,10 @@ public sealed class GalaxyDriver
    ///     null-object resolver (returns null for every name) so seam-injecting tests that
    ///     don't exercise a <c>secret:</c> ref need not supply one.
    /// </param>
+    /// <param name="deployWatchSource">
+    ///     Optional deploy-event stream seam. When supplied, <see cref="InitializeAsync"/> starts the
+    ///     <see cref="DeployWatcher"/> over it (production builds one over the owned repository client).
+    /// </param>
    internal GalaxyDriver(
        string driverInstanceId,
        GalaxyDriverOptions options,
@@ -212,8 +268,10 @@ public sealed class GalaxyDriver
        IGalaxyAlarmAcknowledger? alarmAcknowledger = null,
        IGalaxyAlarmFeed? alarmFeed = null,
        ILogger<GalaxyDriver>? logger = null,
-        ISecretResolver? secretResolver = null)
+        ISecretResolver? secretResolver = null,
+        IGalaxyDeployWatchSource? deployWatchSource = null)
    {
+        _deployWatchSource = deployWatchSource;
        _driverInstanceId = !string.IsNullOrWhiteSpace(driverInstanceId)
            ? driverInstanceId
            : throw new ArgumentException("Driver instance id required.", nameof(driverInstanceId));
@@ -359,28 +417,47 @@ public sealed class GalaxyDriver
        ObjectDisposedException.ThrowIf(_disposed, this);
        if (HasConfigBody(driverConfigJson)) RefuseNonEquivalentConfig(driverConfigJson);

-        // Tests inject seams via the internal ctor; production InitializeAsync builds
-        // the gateway client + session + per-capability runtime components from
-        // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the
-        // test exercises the wired surface without a real gw round-trip).
-        if (_subscriber is null && _dataWriter is null && _hierarchySource is null)
+        await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false);
+        try
        {
-            await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false);
-        }
-        else
-        {
-            _logger.LogDebug(
-                "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped",
-                _driverInstanceId);
-        }
+            ObjectDisposedException.ThrowIf(_disposed, this);

-        await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false);
-        _logger.LogInformation(
-            "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
-            _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName);
-        _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
+            // Tests inject seams via the internal ctor; production InitializeAsync builds
+            // the gateway client + session + per-capability runtime components from
+            // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the
+            // test exercises the wired surface without a real gw round-trip). After a
+            // ShutdownAsync the production seams are null again, so the watchdog's
+            // Shutdown → Initialize genuinely rebuilds the runtime on this same object.
+            if (_subscriber is null && _dataWriter is null && _hierarchySource is null)
+            {
+                await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false);
+            }
+            else
+            {
+                _logger.LogDebug(
+                    "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped",
+                    _driverInstanceId);
+                // The per-platform probe watcher needs only a subscriber, so it runs on the seam path
+                // too — that is what lets the ScanState wiring be exercised without a gateway.
+                if (_subscriber is not null) _probeWatcher ??= BuildProbeWatcher(_subscriber);
+            }
+
+            await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false);
+            _logger.LogInformation(
+                "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
+                _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName);
+            _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
+        }
+        finally
+        {
+            _lifecycleGate.Release();
+        }
    }

+    private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) =>
+        new(subscriber, _hostStatuses, _logger,
+            bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
+
    /// <summary>
    ///     Build the production gw client + session + per-capability runtime components
    ///     from <c>_options</c>. Sets up the reconnect supervisor's reopen / replay
@@ -389,9 +466,32 @@ public sealed class GalaxyDriver
    /// </summary>
    private async Task BuildProductionRuntimeAsync(CancellationToken cancellationToken)
    {
+        try
+        {
+            await BuildProductionRuntimeCoreAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            // A failed connect must not leave a half-built runtime behind: the watchdog retries
+            // InitializeAsync every 10 s, and the seam check above would see the orphaned client +
+            // session as "already built" — or, worse, rebuild on top of them. Release what was
+            // created, and land the failed state before AND after the teardown (IDriver remark).
+            _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message);
+            await TeardownRuntimeAsync(final: false, CancellationToken.None).ConfigureAwait(false);
+            _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message);
+            throw;
+        }
+    }
+
+    private async Task BuildProductionRuntimeCoreAsync(CancellationToken cancellationToken)
+    {
+        _ownsRuntime = true;
        var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false);
        _ownedMxClient = MxGatewayClient.Create(clientOptions);
-        _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger);
+        _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger)
+        {
+            OpenAndRegisterOverrideForTests = SessionOpenOverrideForTests,
+        };
        await _ownedMxSession.ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false);

        // PR 6.1 — wrap the gw-facing seams in tracing decorators so every Subscribe /
@@ -424,9 +524,7 @@ public sealed class GalaxyDriver

        _supervisor.StateChanged += OnSupervisorStateChanged;

-        _probeWatcher = new PerPlatformProbeWatcher(
-            _subscriber, _hostStatuses, _logger,
-            bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
+        _probeWatcher = BuildProbeWatcher(_subscriber);

        // Wire the alarm acknowledger + feed to the live gateway client. Both are
        // session-less — the gateway serves alarms from an always-on central monitor —
@@ -436,6 +534,133 @@ public sealed class GalaxyDriver
            _ownedMxClient.StreamAlarmsAsync, _logger, _options.MxAccess.ClientName);
    }

+    /// <summary>
+    ///     Stop every background loop and release every gateway resource this driver holds — the
+    ///     teardown half shared by <see cref="ShutdownAsync"/> (<paramref name="final"/> = false:
+    ///     a later <see cref="InitializeAsync"/> rebuilds) and <see cref="DisposeAsync"/>
+    ///     (<paramref name="final"/> = true). Before this existed, <c>ShutdownAsync</c> was a
+    ///     no-op and everything below outlived the driver on every respawn, removal and
+    ///     <c>/raw</c> browse: the gateway accumulated live MX sessions and advised items, and
+    ///     this process accumulated channels and stream loops.
+    /// </summary>
+    /// <remarks>
+    ///     Ordering matters:
+    ///     <list type="number">
+    ///         <item>The supervisor goes first and is awaited, so no recovery loop can reopen the
+    ///         session or start a pump while the rest is being released.</item>
+    ///         <item>The probe watcher unsubscribes while the session is still open (its handles
+    ///         die with the session anyway, but a clean release keeps the gateway's advise
+    ///         count honest).</item>
+    ///         <item>Streams (pump, alarm feed, deploy watcher) are cancelled and awaited.</item>
+    ///         <item>The session closes (gateway <c>CloseSession</c>), then both owned channels.</item>
+    ///     </list>
+    ///     Every step is best-effort: one failing release never prevents the next.
+    /// </remarks>
+    private async Task TeardownRuntimeAsync(bool final, CancellationToken cancellationToken)
+    {
+        _runtimeStopping = true;
+        try
+        {
+            var supervisor = _supervisor;
+            _supervisor = null;
+            if (supervisor is not null)
+            {
+                supervisor.StateChanged -= OnSupervisorStateChanged;
+                try { await supervisor.StopAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor stop failed"); }
+            }
+
+            var deployWatcher = _deployWatcher;
+            _deployWatcher = null;
+            if (deployWatcher is not null)
+            {
+                deployWatcher.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded;
+                try { await deployWatcher.StopAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher stop failed"); }
+            }
+
+            var probeWatcher = _probeWatcher;
+            _probeWatcher = null;
+            if (probeWatcher is not null)
+            {
+                try { await probeWatcher.StopAsync(cancellationToken).ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher stop failed"); }
+            }
+
+            try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); }
+            _transportForwarder = null;
+
+            EventPump? pump;
+            lock (_pumpLock) { pump = _eventPump; _eventPump = null; }
+            if (pump is not null)
+            {
+                pump.OnDataChange -= OnPumpDataChange;
+                try { await pump.DisposeAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); }
+            }
+            // Every item handle the registry knows died with the pump's session; the host
+            // re-subscribes (fresh ids) once the next InitializeAsync reports Connected.
+            _subscriptions.Clear();
+
+            // The alarm feed: owned → disposed and nulled so InitializeAsync builds a fresh one;
+            // injected → detached (the injector owns its lifetime) unless this is the final dispose.
+            IGalaxyAlarmFeed? alarmFeed;
+            bool releaseFeed;
+            lock (_alarmFeedLock)
+            {
+                alarmFeed = _alarmFeed;
+                releaseFeed = alarmFeed is not null && (_ownsRuntime || final);
+                if (alarmFeed is not null && _alarmFeedWired)
+                {
+                    alarmFeed.OnAlarmTransition -= OnAlarmFeedTransition;
+                    _alarmFeedWired = false;
+                }
+                if (releaseFeed) _alarmFeed = null;
+            }
+            if (releaseFeed)
+            {
+                try { await alarmFeed!.DisposeAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); }
+            }
+
+            if (_ownedMxSession is not null)
+            {
+                try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); }
+                _ownedMxSession = null;
+            }
+
+            if (_ownedMxClient is not null)
+            {
+                try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); }
+                _ownedMxClient = null;
+            }
+
+            if (_ownedRepositoryClient is not null)
+            {
+                try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); }
+                catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); }
+                _ownedRepositoryClient = null;
+            }
+
+            if (_ownsRuntime || final)
+            {
+                // Null the seams so the next InitializeAsync rebuilds (production) — injected seams
+                // are only dropped on the final dispose.
+                _subscriber = null;
+                _dataWriter = null;
+                _hierarchySource = null;
+                _alarmAcknowledger = null;
+                _ownsRuntime = false;
+            }
+        }
+        finally
+        {
+            _runtimeStopping = false;
+        }
+    }
+
    /// <summary>
    ///     Reopen callback for <see cref="ReconnectSupervisor"/>: recreate the gw session. After a
    ///     gateway restart the existing session handle is faulted / stale but non-null, so a plain
@@ -465,6 +690,9 @@ public sealed class GalaxyDriver
    /// </summary>
    private async Task ReplayAsync(CancellationToken cancellationToken)
    {
+        // A teardown cancels the supervisor's cycle between reopen and replay; a replay that ran on
+        // anyway would start a fresh pump underneath the shutdown.
+        cancellationToken.ThrowIfCancellationRequested();
        if (_subscriber is null) return;
        var entries = _subscriptions.SnapshotEntries();
        if (entries.Count == 0) return;
@@ -581,9 +809,9 @@ public sealed class GalaxyDriver
        }

        // EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the
-        // recreate if the driver is being disposed — Dispose already tore the pump down
-        // and a fresh one here would leak past the driver's lifetime.
-        if (_disposed) return;
+        // recreate if the driver is being disposed or its runtime torn down — the teardown
+        // already released the pump and a fresh one here would leak past it.
+        if (_disposed || _runtimeStopping) return;
        EnsureEventPumpStarted();
    }

@@ -631,28 +859,53 @@ public sealed class GalaxyDriver
    private async Task StartDeployWatcherAsync(CancellationToken cancellationToken)
    {
        if (!_options.Repository.WatchDeployEvents) return;
-        if (_ownedRepositoryClient is null && _hierarchySource is null) return;

-        // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand).
-        // If discovery hasn't run yet, build the client here so the watcher has a target.
-        // Build under a null-check (not ??=) so if BuildDefaultHierarchySourceAsync later
-        // runs it reuses this client rather than overwriting the field and leaking the
-        // first instance — the client-options build is now async (secret: arm).
-        _ownedRepositoryClient ??= ZB.MOM.WW.MxGateway.Client.GalaxyRepositoryClient.Create(
-            await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false));
+        var source = _deployWatchSource;
+        if (source is null)
+        {
+            // Seam mode without an injected watch source has no gateway to watch. (Production
+            // always owns its runtime here.) The old guard returned whenever BOTH the repository
+            // client and the hierarchy source were null — which on a production runtime instance
+            // is every first InitializeAsync, since discovery runs on the /raw picker's throwaway
+            // instance, not here — so the watcher never started and the re-browse chip never
+            // fired for a live driver.
+            if (!_ownsRuntime) return;

-        var source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient);
-        _deployWatcher = new DeployWatcher(source, _logger);
-        _deployWatcher.OnRediscoveryNeeded += (_, args) => OnRediscoveryNeeded?.Invoke(this, args);
+            // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand).
+            // Build under a null-check (not an overwrite) so if BuildDefaultHierarchySourceAsync
+            // later runs it reuses this client rather than leaking the first instance.
+            _ownedRepositoryClient ??= GalaxyRepositoryClient.Create(
+                await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false));
+            source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient);
+        }
+
+        // One watcher per driver: a previous one (an InitializeAsync without a ShutdownAsync in
+        // between) is stopped before its replacement starts — every live InitializeAsync used to
+        // stack another watch-stream loop, each raising OnRediscoveryNeeded per real deploy.
+        var previous = _deployWatcher;
+        _deployWatcher = null;
+        if (previous is not null)
+        {
+            previous.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded;
+            try { await previous.StopAsync().ConfigureAwait(false); }
+            catch (Exception ex) { _logger.LogWarning(ex, "Previous DeployWatcher stop failed — replacing it anyway."); }
+        }
+
+        var watcher = new DeployWatcher(source, _logger);
+        watcher.OnRediscoveryNeeded += OnDeployWatcherRediscoveryNeeded;
+        _deployWatcher = watcher;

        // StartAsync schedules the background loop and returns Task.CompletedTask immediately.
        // It throws InvalidOperationException synchronously if called twice (programming error).
        // Don't discard the return value — observe any synchronous throw.
-        var startTask = _deployWatcher.StartAsync(CancellationToken.None);
+        var startTask = watcher.StartAsync(CancellationToken.None);
        // The task is already completed (StartAsync is synchronous); surface any synchronous fault.
        if (startTask.IsFaulted) startTask.GetAwaiter().GetResult();
    }

+    private void OnDeployWatcherRediscoveryNeeded(object? sender, RediscoveryEventArgs args) =>
+        OnRediscoveryNeeded?.Invoke(this, args);
+
    /// <inheritdoc />
    public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
    {
@@ -731,13 +984,32 @@ public sealed class GalaxyDriver
        && a.Reconnect == b.Reconnect
        && a.ProbeTimeoutSeconds == b.ProbeTimeoutSeconds;

+    /// <summary>
+    ///     Stop the driver and release the gateway runtime: supervisor, deploy watcher, probe
+    ///     watcher, event pump, alarm feed, MX session (gateway <c>CloseSession</c>) and both owned
+    ///     gRPC channels. The seams are nulled afterwards, so a later <see cref="InitializeAsync"/>
+    ///     on this same object — the host's comms-loss recovery and the way <c>ReinitializeAsync</c>
+    ///     is sequenced — rebuilds a fresh runtime instead of reusing a dead one. The driver object
+    ///     itself stays usable; <see cref="DisposeAsync"/> is the final, idempotent teardown.
+    /// </summary>
    /// <inheritdoc />
-    public Task ShutdownAsync(CancellationToken cancellationToken)
+    public async Task ShutdownAsync(CancellationToken cancellationToken)
    {
-        if (_disposed) return Task.CompletedTask;
+        if (_disposed) return;
        _logger.LogInformation("GalaxyDriver {InstanceId} shutting down", _driverInstanceId);
-        _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
-        return Task.CompletedTask;
+        await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false);
+        try
+        {
+            if (_disposed) return;
+            await TeardownRuntimeAsync(final: false, cancellationToken).ConfigureAwait(false);
+            // Terminal health lands AFTER the teardown (IDriver remark): a loop unwinding during the
+            // release could otherwise overwrite it with the state it was reporting when cancelled.
+            _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
+        }
+        finally
+        {
+            _lifecycleGate.Release();
+        }
    }

    /// <inheritdoc />
@@ -1192,7 +1464,7 @@ public sealed class GalaxyDriver
    /// </remarks>
    private void OnEventPumpReplayGap(ReplayGap gap)
    {
-        if (_subscriber is null) return;
+        if (_subscriber is null || _runtimeStopping) return;

        if (!_options.Reconnect.ReplayOnSessionLost)
        {
@@ -1249,6 +1521,14 @@ public sealed class GalaxyDriver
    /// </summary>
    private void OnEventPumpStreamFault(Exception cause)
    {
+        if (_runtimeStopping)
+        {
+            // The pump is being torn down with the rest of the runtime — a fault it reports on the
+            // way out is shutdown noise, not something to recover from.
+            _logger.LogDebug(cause, "GalaxyDriver {InstanceId} event stream ended during runtime teardown.", _driverInstanceId);
+            return;
+        }
+
        var supervisor = _supervisor;
        if (supervisor is null)
        {
@@ -1517,54 +1797,19 @@ public sealed class GalaxyDriver
    public async ValueTask DisposeAsync()
    {
        if (_disposed) return;
-        _disposed = true;
-
-        // Synchronous sub-components first — none of these block.
-        try { _deployWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher dispose failed"); }
-        try { _supervisor?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor dispose failed"); }
-        try { _probeWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher dispose failed"); }
-        try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); }
-
-        // Async sub-components: await each so we don't block a thread-pool thread
-        // on a slow shutdown (e.g. EventPump draining its channel, gRPC stream closing).
-        EventPump? pump;
-        lock (_pumpLock) { pump = _eventPump; _eventPump = null; }
-        if (pump is not null)
+        await _lifecycleGate.WaitAsync().ConfigureAwait(false);
+        try
        {
-            try { await pump.DisposeAsync().ConfigureAwait(false); }
-            catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); }
+            if (_disposed) return;
+            _disposed = true;
+            // Same release sequence as ShutdownAsync, plus the injected seams this instance will
+            // never need again. Idempotent: a ShutdownAsync before this finds nothing left to do.
+            await TeardownRuntimeAsync(final: true, CancellationToken.None).ConfigureAwait(false);
        }
-
-        IGalaxyAlarmFeed? alarmFeed;
-        lock (_alarmFeedLock) { alarmFeed = _alarmFeed; _alarmFeed = null; }
-        if (alarmFeed is not null)
+        finally
        {
-            try { await alarmFeed.DisposeAsync().ConfigureAwait(false); }
-            catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); }
+            _lifecycleGate.Release();
        }
-
-        if (_ownedMxSession is not null)
-        {
-            try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); }
-            catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); }
-            _ownedMxSession = null;
-        }
-
-        if (_ownedMxClient is not null)
-        {
-            try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); }
-            catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); }
-            _ownedMxClient = null;
-        }
-
-        if (_ownedRepositoryClient is not null)
-        {
-            try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); }
-            catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); }
-            _ownedRepositoryClient = null;
-        }
-
-        _hierarchySource = null;
    }

    /// <summary>
@@ -196,20 +196,28 @@ public sealed class PerPlatformProbeWatcher : IDisposable
        };
    }

-    /// <summary>Disposes the probe watcher and unsubscribes all tracked platforms.</summary>
-    public void Dispose()
+    /// <summary>
+    ///     Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway
+    ///     (best-effort — a transport error is logged and swallowed). The async counterpart of
+    ///     <see cref="Dispose"/>; the driver's <c>ShutdownAsync</c> uses it so the unsubscribe
+    ///     round-trip is awaited rather than blocked on. Idempotent.
+    /// </summary>
+    /// <param name="cancellationToken">Bounds the unsubscribe round-trip.</param>
+    /// <returns>A task that completes once the handles have been released (or the attempt gave up).</returns>
+    public async Task StopAsync(CancellationToken cancellationToken)
    {
        if (_disposed) return;
        _disposed = true;

-        // Best-effort unsubscribe everything we know about. Run synchronously through
-        // GetAwaiter().GetResult() since Dispose is sync; transport errors are swallowed.
        var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray();
        _itemHandlesByPlatform.Clear();
        if (liveHandles.Length > 0)
        {
-            try { _subscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None).GetAwaiter().GetResult(); }
-            catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher dispose unsubscribe failed"); }
+            try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); }
+            catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); }
        }
    }
+
+    /// <summary>Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip).</summary>
+    public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult();
 }
@@ -187,9 +187,11 @@ internal sealed class EventPump : IAsyncDisposable
                NotifyStreamFault(cause);
            }
        }
-        catch (OperationCanceledException) when (ct.IsCancellationRequested)
+        catch (Exception) when (ct.IsCancellationRequested)
        {
-            // Clean shutdown — no log.
+            // Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled)
+            // rather than OperationCanceledException; either way the pump was told to stop, so this
+            // is not a transport fault and must not wake the reconnect supervisor mid-teardown.
        }
        catch (Exception ex)
        {
@@ -62,6 +62,16 @@ public sealed class ReconnectSupervisor : IDisposable
    private Task? _recoveryLoop;
    private CancellationTokenSource? _loopCts;
    private bool _disposed;
+    private int _cyclesStarted;
+    private int _cyclesReleased;
+
+    /// <summary>Recovery cycles started so far (one per <see cref="ReportTransportFailure"/> that left Healthy). Test seam.</summary>
+    internal int CyclesStarted => Volatile.Read(ref _cyclesStarted);
+
+    /// <summary>Recovery cycles whose <see cref="CancellationTokenSource"/> has been disposed. Equals
+    /// <see cref="CyclesStarted"/> once every loop has exited — a cycle's CTS used to be released only
+    /// if it happened to be the LAST one when <see cref="Dispose"/> ran. Test seam.</summary>
+    internal int CyclesReleased => Volatile.Read(ref _cyclesReleased);

    /// <summary>Fires after every state transition.</summary>
    public event EventHandler<StateTransition>? StateChanged;
@@ -136,8 +146,13 @@ public sealed class ReconnectSupervisor : IDisposable

            TransitionLocked(State.TransportLost, cause.Message);

-            _loopCts = new CancellationTokenSource();
-            _recoveryLoop = Task.Run(() => RecoveryLoopAsync(_loopCts.Token));
+            // Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's
+            // finally). Before, every cycle allocated a fresh CTS and only the last one was ever
+            // disposed — one leaked CancellationTokenSource per outage for the driver's lifetime.
+            var cts = new CancellationTokenSource();
+            _loopCts = cts;
+            Interlocked.Increment(ref _cyclesStarted);
+            _recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts));
        }
    }

@@ -156,7 +171,26 @@ public sealed class ReconnectSupervisor : IDisposable
        }
    }

-    private async Task RecoveryLoopAsync(CancellationToken ct)
+    private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts)
+    {
+        try
+        {
+            await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false);
+        }
+        finally
+        {
+            // Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may
+            // race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object.
+            lock (_stateLock)
+            {
+                if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null;
+            }
+            cycleCts.Dispose();
+            Interlocked.Increment(ref _cyclesReleased);
+        }
+    }
+
+    private async Task RecoveryLoopCoreAsync(CancellationToken ct)
    {
        var attempt = 0;
        while (!ct.IsCancellationRequested)
@@ -240,8 +274,13 @@ public sealed class ReconnectSupervisor : IDisposable
        }
    }

-    /// <summary>Disposes the supervisor and cancels any in-flight recovery loop.</summary>
-    public void Dispose()
+    /// <summary>
+    ///     Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's
+    ///     thread. The async counterpart of <see cref="Dispose"/>; the driver's <c>ShutdownAsync</c>
+    ///     uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent.
+    /// </summary>
+    /// <returns>A task that completes once the recovery loop has exited.</returns>
+    public async Task StopAsync()
    {
        if (_disposed) return;
        _disposed = true;
@@ -250,13 +289,18 @@ public sealed class ReconnectSupervisor : IDisposable
        Task? loop;
        lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }

-        cts?.Cancel();
+        // The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
+        // throws on Cancel, and a disposed one is a no-op on Dispose.
+        try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ }
        if (loop is not null)
        {
-            try { loop.GetAwaiter().GetResult(); } catch { /* shutdown */ }
+            try { await loop.ConfigureAwait(false); } catch { /* shutdown */ }
        }
        cts?.Dispose();
    }
+
+    /// <summary>Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits).</summary>
+    public void Dispose() => StopAsync().GetAwaiter().GetResult();
 }

 /// <summary>
@@ -100,6 +100,17 @@ internal sealed class SubscriptionRegistry
    }


+    /// <summary>
+    ///     Drop every subscription and the whole reverse map. Used by the driver's runtime teardown:
+    ///     once the gateway session is closed every item handle in here is dead, and the host
+    ///     re-issues its subscriptions (fresh ids) after the next <c>InitializeAsync</c>.
+    /// </summary>
+    public void Clear()
+    {
+        _bySubscriptionId.Clear();
+        _subscribersByItemHandle.Clear();
+    }
+
    /// <summary>Snapshot every active binding for diagnostic output.</summary>
    /// <returns>A flattened list of every tag binding across all tracked subscriptions.</returns>
    public IReadOnlyList<TagBinding> SnapshotAllBindings() =>