fix(communication): cancel-only CTS replacement, deferred channel disposal, single-endpoint debug stream (arch-review 2026-08-19 P3-1 + P3-2 + P3-3)

src/ZB.MOM.WW.ScadaBridge.Communication/DebugStreamService.cs

@@ -108,10 +108,22 @@ public class DebugStreamService : IDisposable
 
             instanceUniqueName = instance.UniqueName;
             siteIdentifier = site.SiteIdentifier;
-            grpcNodeAAddress = site.GrpcNodeAAddress
-                ?? throw new InvalidOperationException($"Site {siteIdentifier} has no GrpcNodeAAddress configured.");
-            grpcNodeBAddress = site.GrpcNodeBAddress
-                ?? throw new InvalidOperationException($"Site {siteIdentifier} has no GrpcNodeBAddress configured.");
+
+            // Single-endpoint site (arch-review 2026-08-19 P3-3, applying the same N9
+            // degeneration SiteAlarmLiveCacheService.ResolveSiteAsync already uses): reuse the
+            // sole configured endpoint for both slots, so the bridge actor's NodeA↔NodeB
+            // failover flip degenerates to a reconnect against the same node. Requiring BOTH
+            // addresses made Debug View the only surface that refused a one-node site.
+            var nodeA = site.GrpcNodeAAddress;
+            var nodeB = site.GrpcNodeBAddress;
+            if (string.IsNullOrWhiteSpace(nodeA) && string.IsNullOrWhiteSpace(nodeB))
+            {
+                throw new InvalidOperationException(
+                    $"Site {siteIdentifier} has no gRPC node addresses configured.");
+            }
+
+            grpcNodeAAddress = !string.IsNullOrWhiteSpace(nodeA) ? nodeA! : nodeB!;
+            grpcNodeBAddress = !string.IsNullOrWhiteSpace(nodeB) ? nodeB! : nodeA!;
         }
 
         var sessionId = Guid.NewGuid().ToString("N");

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/SitePairChannelProvider.cs

@@ -53,6 +53,14 @@ public sealed class SitePairChannelProvider : IDisposable
     private static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(60);
     private static readonly TimeSpan FailbackSweepInterval = TimeSpan.FromSeconds(1);
 
+    /// <summary>
+    /// How long a replaced/removed channel is PARKED before it is disposed (arch-review
+    /// 2026-08-19 P3-2). Sized to exceed the longest command deadline
+    /// (<c>CommunicationOptions.DeploymentTimeout</c> and friends, minutes at most), so no call
+    /// that was already in flight when the channel was replaced can still be using it.
+    /// </summary>
+    internal static readonly TimeSpan ChannelDisposalGrace = TimeSpan.FromMinutes(5);
+
     private readonly ISitePskProvider _pskProvider;
     private readonly CommunicationOptions _options;
     private readonly ILogger<SitePairChannelProvider> _logger;
@@ -60,6 +68,14 @@ public sealed class SitePairChannelProvider : IDisposable
     private readonly Func<GrpcChannel, CancellationToken, Task<bool>> _reachabilityProbe;
 
     private readonly ConcurrentDictionary<string, SitePair> _sites = new(StringComparer.Ordinal);
+
+    /// <summary>
+    /// Channels replaced by <see cref="UpdateSite"/> or dropped by <see cref="RemoveSite"/>,
+    /// each with the instant it becomes safe to dispose. FIFO order is disposal order because
+    /// the grace window is constant.
+    /// </summary>
+    private readonly ConcurrentQueue<(GrpcChannel Channel, DateTime DisposeAfterUtc)> _graveyard = new();
+
     private readonly CancellationTokenSource _shutdown = new();
     private Task? _failbackLoop;
     private readonly object _loopGate = new();
@@ -98,8 +114,9 @@ public sealed class SitePairChannelProvider : IDisposable
 
     /// <summary>
     /// (Re)builds a site's channel pair from its gRPC node endpoints. A no-op when neither endpoint
-    /// changed. Disposes and rebuilds a channel whose endpoint changed, and resets stickiness to the
-    /// preferred node when the pair is (re)created. Called on the DB refresh tick.
+    /// changed. Parks (see <see cref="ChannelDisposalGrace"/>) and rebuilds a channel whose endpoint
+    /// changed, and resets stickiness to the preferred node when the pair is (re)created. Called on
+    /// the DB refresh tick.
     /// </summary>
     /// <param name="siteId">Site identifier.</param>
     /// <param name="nodeAEndpoint">NodeA gRPC base address, or null when unconfigured.</param>
@@ -107,6 +124,7 @@ public sealed class SitePairChannelProvider : IDisposable
     public void UpdateSite(string siteId, string? nodeAEndpoint, string? nodeBEndpoint)
     {
         ArgumentException.ThrowIfNullOrWhiteSpace(siteId);
+        SweepGraveyard();
 
         var pair = _sites.GetOrAdd(siteId, id => new SitePair(id));
         lock (pair.Gate)
@@ -120,13 +138,13 @@ public sealed class SitePairChannelProvider : IDisposable
 
             if (changedA)
             {
-                pair.ChannelA?.Dispose();
+                Park(pair.ChannelA);
                 pair.ChannelA = string.IsNullOrWhiteSpace(nodeAEndpoint) ? null : BuildChannel(nodeAEndpoint, siteId);
                 pair.NodeAEndpoint = nodeAEndpoint;
             }
             if (changedB)
             {
-                pair.ChannelB?.Dispose();
+                Park(pair.ChannelB);
                 pair.ChannelB = string.IsNullOrWhiteSpace(nodeBEndpoint) ? null : BuildChannel(nodeBEndpoint, siteId);
                 pair.NodeBEndpoint = nodeBEndpoint;
             }
@@ -140,7 +158,10 @@ public sealed class SitePairChannelProvider : IDisposable
         }
     }
 
-    /// <summary>Disposes a removed site's channels and invalidates its cached preshared key.</summary>
+    /// <summary>
+    /// Parks a removed site's channels for deferred disposal (see <see cref="ChannelDisposalGrace"/>)
+    /// and invalidates its cached preshared key.
+    /// </summary>
     /// <param name="siteId">Site identifier to remove.</param>
     public void RemoveSite(string siteId)
     {
@@ -149,12 +170,14 @@ public sealed class SitePairChannelProvider : IDisposable
             return;
         }
 
+        SweepGraveyard();
+
         if (_sites.TryRemove(siteId, out var pair))
         {
             lock (pair.Gate)
             {
-                pair.ChannelA?.Dispose();
-                pair.ChannelB?.Dispose();
+                Park(pair.ChannelA);
+                Park(pair.ChannelB);
                 pair.ChannelA = null;
                 pair.ChannelB = null;
             }
@@ -284,6 +307,7 @@ public sealed class SitePairChannelProvider : IDisposable
         {
             while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false))
             {
+                SweepGraveyard();
                 var now = DateTime.UtcNow;
                 foreach (var kvp in _sites)
                 {
@@ -317,6 +341,67 @@ public sealed class SitePairChannelProvider : IDisposable
         }
     }
 
+    /// <summary>
+    /// Parks a channel that is no longer reachable through <see cref="_sites"/> instead of
+    /// disposing it inline (arch-review 2026-08-19 P3-2).
+    /// <para>
+    /// <b>Why deferred.</b> <see cref="ExecuteAsync{T}"/> reads the channel under
+    /// <c>pair.Gate</c> but invokes the RPC OUTSIDE it, so an inline
+    /// <c>Dispose</c> from the 60 s DB address refresh (or an admin edit) aborted whatever was
+    /// in flight — a multi-minute <c>DeployArtifactsCommand</c> included — with
+    /// <see cref="ObjectDisposedException"/> or <see cref="StatusCode.Cancelled"/>. Neither is
+    /// <see cref="StatusCode.Unavailable"/>, so the cross-node retry deliberately never engaged
+    /// and the command simply failed.
+    /// </para>
+    /// <para>
+    /// <b>Why a timed park rather than reference counting.</b> A refcount would have to be taken
+    /// and released around every <c>ExecuteAsync</c> call — bookkeeping on the hot path, plus a
+    /// quiescence handshake — to buy nothing a fixed window does not already give: the window
+    /// only has to outlast the longest command deadline. It also matches the precedent set by
+    /// <see cref="SiteStreamGrpcClientFactory"/>, which likewise keeps a superseded endpoint's
+    /// channel alive rather than disposing it under a concurrent session. Cost is a handful of
+    /// idle HTTP/2 channels for at most <see cref="ChannelDisposalGrace"/>.
+    /// </para>
+    /// </summary>
+    /// <param name="channel">The channel to park; null is ignored.</param>
+    private void Park(GrpcChannel? channel)
+    {
+        if (channel is not null)
+        {
+            _graveyard.Enqueue((channel, DateTime.UtcNow + ChannelDisposalGrace));
+        }
+    }
+
+    /// <summary>
+    /// Disposes every parked channel whose grace window has elapsed. Called from the DB refresh
+    /// path (<see cref="UpdateSite"/>/<see cref="RemoveSite"/>) and from the failback loop, so a
+    /// park always drains even on a provider that never runs a call again.
+    /// </summary>
+    /// <param name="force">True to dispose every parked channel regardless of its window (shutdown, and tests).</param>
+    internal void SweepGraveyard(bool force = false)
+    {
+        var now = DateTime.UtcNow;
+        while (_graveyard.TryPeek(out var parked) && (force || now >= parked.DisposeAfterUtc))
+        {
+            if (!_graveyard.TryDequeue(out parked))
+            {
+                break;
+            }
+
+            try
+            {
+                parked.Channel.Dispose();
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Disposing a parked gRPC channel faulted");
+            }
+        }
+    }
+
+    /// <summary>Test/diagnostic accessor: how many replaced channels are awaiting deferred disposal.</summary>
+    internal int ParkedChannelCount => _graveyard.Count;
+
     private void FlipTo(SitePair pair, bool toIsA)
     {
         lock (pair.Gate)
@@ -396,6 +481,10 @@ public sealed class SitePairChannelProvider : IDisposable
             }
         }
         _sites.Clear();
+
+        // Process shutdown — nothing can still be in flight, so drain the graveyard regardless
+        // of the remaining grace on each entry.
+        SweepGraveyard(force: true);
     }
 
     /// <summary>Per-site mutable channel state, guarded by <see cref="Gate"/>.</summary>

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/SiteStreamGrpcClient.cs

@@ -138,7 +138,18 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     /// Registers a subscription's CancellationTokenSource for a correlation ID.
     /// If an entry already exists for that correlation ID (a reconnect race where two
     /// <see cref="SubscribeAsync"/> calls briefly share an ID), the prior CTS is
-    /// cancelled and disposed so it cannot leak. Internal for testability.
+    /// <b>cancelled only</b> — never disposed from here. Internal for testability.
+    /// <para>
+    /// <b>Cancel-only is the contract</b> (arch-review 2026-08-19 P3-1, the client half of the
+    /// row-47 class): the prior handler's <see cref="ConsumeStreamAsync"/> is still running and
+    /// still reads <c>cts.Token</c> (opening the call, <c>ReadAllAsync</c>, the
+    /// <c>IsCancellationRequested</c> checks), so disposing it from the REPLACING path throws
+    /// <see cref="ObjectDisposedException"/> inside the prior stream — and, worse, out of the
+    /// <c>prior.Cancel()</c> call itself into the NEW <c>SubscribeAsync</c>, where it reads as a
+    /// fault of the current generation and burns a reconnect. Disposal belongs to the OWNING
+    /// path: <see cref="SubscribeAsync"/>/<see cref="SubscribeSiteAsync"/> dispose the CTS they
+    /// created once their stream loop has fully unwound.
+    /// </para>
     /// </summary>
     /// <param name="correlationId">Unique identifier for the subscription.</param>
     /// <param name="cts">CancellationTokenSource for managing the subscription lifecycle.</param>
@@ -146,12 +157,30 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     {
         if (_subscriptions.TryGetValue(correlationId, out var prior) && !ReferenceEquals(prior, cts))
         {
-            prior.Cancel();
-            prior.Dispose();
+            CancelQuietly(prior);
         }
         _subscriptions[correlationId] = cts;
     }
 
+    /// <summary>
+    /// Cancels a subscription token source, swallowing <see cref="ObjectDisposedException"/>.
+    /// A CTS whose owning stream loop has already unwound and disposed it is simply a
+    /// subscription that no longer needs cancelling; the caller must never see that race as a
+    /// failure (arch-review 2026-08-19 P3-1).
+    /// </summary>
+    /// <param name="cts">The token source to cancel.</param>
+    private static void CancelQuietly(CancellationTokenSource cts)
+    {
+        try
+        {
+            cts.Cancel();
+        }
+        catch (ObjectDisposedException)
+        {
+            // Already disposed by its owning stream loop — nothing left to cancel.
+        }
+    }
+
     /// <summary>
     /// Removes the subscription entry for a correlation ID only if the stored CTS is
     /// exactly the one supplied. A racing replacement stream may already own the slot,
@@ -213,13 +242,23 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
                 onEvent(domainEvent);
         }
 
-        await ConsumeStreamAsync(
-            correlationId,
-            cts,
-            () => _client.SubscribeInstance(request, cancellationToken: cts.Token),
-            frame => ForEachEvent(frame, Deliver),
-            onError,
-            onCompleted);
+        // This call OWNS `cts` — it created it, so it disposes it once the stream loop has
+        // fully unwound (P3-1). No other path may dispose it while this one is still reading
+        // cts.Token; a replacing subscription cancels only.
+        try
+        {
+            await ConsumeStreamAsync(
+                correlationId,
+                cts,
+                () => _client.SubscribeInstance(request, cancellationToken: cts.Token),
+                frame => ForEachEvent(frame, Deliver),
+                onError,
+                onCompleted);
+        }
+        finally
+        {
+            cts.Dispose();
+        }
     }
 
     /// <summary>
@@ -288,14 +327,22 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
                 onAlarmEvent(alarm);
         }
 
-        await ConsumeStreamAsync(
-            correlationId,
-            cts,
-            () => _client.SubscribeSite(request, cancellationToken: cts.Token),
-            frame => ForEachEvent(frame, Deliver),
-            onError,
-            onCompleted,
-            onConnected);
+        // Owns `cts` — see SubscribeAsync.
+        try
+        {
+            await ConsumeStreamAsync(
+                correlationId,
+                cts,
+                () => _client.SubscribeSite(request, cancellationToken: cts.Token),
+                frame => ForEachEvent(frame, Deliver),
+                onError,
+                onCompleted,
+                onConnected);
+        }
+        finally
+        {
+            cts.Dispose();
+        }
     }
 
     /// <summary>
@@ -449,15 +496,17 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     }
 
     /// <summary>
-    /// Cancels an active subscription by correlation ID.
+    /// Cancels an active subscription by correlation ID. Cancel-only: the subscription's own
+    /// <see cref="SubscribeAsync"/>/<see cref="SubscribeSiteAsync"/> call disposes the token
+    /// source once its stream loop unwinds (P3-1). Cancelling an entry whose owner already
+    /// disposed it — or unsubscribing after the client was disposed — is a silent no-op.
     /// </summary>
     /// <param name="correlationId">Unique identifier of the subscription to cancel.</param>
     public virtual void Unsubscribe(string correlationId)
     {
         if (_subscriptions.TryRemove(correlationId, out var cts))
         {
-            cts.Cancel();
-            cts.Dispose();
+            CancelQuietly(cts);
         }
     }
 
@@ -601,17 +650,23 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     };
 
     /// <summary>
-    /// Releases all subscription CancellationTokenSources and the underlying
-    /// gRPC channel. All teardown here is synchronous (CTS disposal and
-    /// <see cref="GrpcChannel.Dispose"/>), so a synchronous <see cref="Dispose"/>
-    /// can release everything without sync-over-async blocking.
+    /// Cancels every live subscription and releases the underlying gRPC channel. All teardown
+    /// here is synchronous (CTS cancellation and <see cref="GrpcChannel.Dispose"/>), so a
+    /// synchronous <see cref="Dispose"/> can release everything without sync-over-async blocking.
+    /// <para>
+    /// Cancel-only, per-entry guarded (P3-1). The token sources are owned by their
+    /// <see cref="SubscribeAsync"/>/<see cref="SubscribeSiteAsync"/> calls, which dispose them
+    /// as their loops unwind in response to this cancellation; disposing them here raced those
+    /// loops, and one <see cref="ObjectDisposedException"/> aborted the whole foreach — which in
+    /// turn aborted <see cref="SiteStreamGrpcClientFactory.DisposeAsync"/>'s client loop and left
+    /// every later client (and its channel) undisposed.
+    /// </para>
     /// </summary>
     private void ReleaseResources()
     {
         foreach (var cts in _subscriptions.Values)
         {
-            cts.Cancel();
-            cts.Dispose();
+            CancelQuietly(cts);
         }
         _subscriptions.Clear();
 

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/SiteStreamGrpcClientFactory.cs

@@ -120,7 +120,7 @@ public class SiteStreamGrpcClientFactory : IAsyncDisposable, IDisposable
         foreach (var key in _clients.Keys.Where(k => k.Site == siteIdentifier).ToList())
         {
             if (_clients.TryRemove(key, out var client))
-                await client.DisposeAsync();
+                await DisposeClientAsync(client, key);
         }
 
         // Drop the cached preshared key too, so a site removed and re-added under the same
@@ -129,16 +129,39 @@ public class SiteStreamGrpcClientFactory : IAsyncDisposable, IDisposable
     }
 
     /// <summary>
-    /// Asynchronously disposes all cached clients and clears the cache.
+    /// Asynchronously disposes all cached clients and clears the cache. Each client's disposal
+    /// is fault-isolated: one throwing client must not abort the loop and strand every later
+    /// client's channel undisposed (arch-review 2026-08-19 P3-1).
     /// </summary>
     /// <returns>A value task that completes when all clients have been disposed.</returns>
     public async ValueTask DisposeAsync()
     {
-        foreach (var client in _clients.Values)
+        foreach (var entry in _clients)
+        {
+            await DisposeClientAsync(entry.Value, entry.Key);
+        }
+        _clients.Clear();
+    }
+
+    /// <summary>
+    /// Disposes one client, logging and swallowing any failure so the caller's disposal loop
+    /// continues to the remaining clients.
+    /// </summary>
+    /// <param name="client">The client to dispose.</param>
+    /// <param name="key">Cache key of the client, for diagnostics.</param>
+    /// <returns>A task that completes when the client has been disposed (or its failure logged).</returns>
+    private async Task DisposeClientAsync(SiteStreamGrpcClient client, (string Site, string Endpoint) key)
+    {
+        try
         {
             await client.DisposeAsync();
         }
-        _clients.Clear();
+        catch (Exception ex)
+        {
+            _loggerFactory.CreateLogger<SiteStreamGrpcClientFactory>().LogWarning(
+                ex, "Disposing the site stream gRPC client for site {SiteId} endpoint {Endpoint} failed",
+                key.Site, key.Endpoint);
+        }
     }
 
     /// <summary>
@@ -147,12 +170,22 @@ public class SiteStreamGrpcClientFactory : IAsyncDisposable, IDisposable
     /// a stall/deadlock risk during host shutdown). Each
     /// <see cref="SiteStreamGrpcClient"/> releases all of its resources
     /// synchronously, so we dispose them directly with no async path.
+    /// Fault-isolated per client, exactly as <see cref="DisposeAsync"/> is.
     /// </summary>
     public void Dispose()
     {
-        foreach (var client in _clients.Values)
+        foreach (var entry in _clients)
         {
-            client.Dispose();
+            try
+            {
+                entry.Value.Dispose();
+            }
+            catch (Exception ex)
+            {
+                _loggerFactory.CreateLogger<SiteStreamGrpcClientFactory>().LogWarning(
+                    ex, "Disposing the site stream gRPC client for site {SiteId} endpoint {Endpoint} failed",
+                    entry.Key.Site, entry.Key.Endpoint);
+            }
         }
         _clients.Clear();
     }