fix(communication): fail a site command over within the call when the stopped node never received it (#136)

The first mutation (instance deploy) to a site pair whose sticky node was down
returned OUTCOME_UNKNOWN: SitePairChannelProvider.ExecuteAsync treated every
Unavailable on a non-retry-safe command as possibly-applied unless the server
sent the not-dispatched trailer, flipped stickiness, and threw. The retry then
succeeded on the live peer.

A connect-phase failure provably never sent the request, so it is now treated
like the pre-dispatch rejection: GrpcRetrySafety.WasNeverSent recognises
Grpc.Net.Client's subchannel connect failure ("Error connecting to subchannel."
with a SocketException cause) and an HttpRequestException name-resolution /
connection error with a connect-only socket error. Anything that may have been
delivered (reset, premature end, GOAWAY, server Unavailable) stays
OUTCOME_UNKNOWN and is not replayed. The site->central reconcile path uses the
same ProvablyNotApplied rule.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/GrpcCentralTransport.cs

@@ -28,7 +28,8 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 /// <para>
 /// <b>Cross-node retry is operation-specific.</b> Read-only status and idempotent notification,
 /// audit and sequence-guarded health ingestion may retry transient transport failures. Reconcile
-/// mints tokens and retries only an explicit server pre-dispatch rejection. Ambiguous reconcile
+/// mints tokens and retries only an explicit server pre-dispatch rejection or a request that was
+/// provably never sent (connect-phase failure, ScadaBridge#136). Ambiguous reconcile
 /// failures return OUTCOME_UNKNOWN. Both attempts share one absolute deadline.
 /// </para>
 /// <para>
@@ -182,12 +183,13 @@ public sealed class GrpcCentralTransport : ICentralTransport
         catch (RpcException ex) when (IsConnectFailure(ex))
         {
             _channels.ReportUnavailable(index);
-            if (!retrySafe && !GrpcRetrySafety.WasNotDispatched(ex))
+            if (!retrySafe && !GrpcRetrySafety.ProvablyNotApplied(ex))
             {
                 replyTo.Tell(new AkkaStatus.Failure(GrpcRetrySafety.OutcomeUnknown(ex)));
                 return;
             }
-            // Retry only a read/idempotent ingest or an explicit server pre-dispatch rejection.
+            // Retry only a read/idempotent ingest, an explicit server pre-dispatch rejection, or a
+            // connect-phase failure that never sent the request (ScadaBridge#136).
             var (retryIndex, retryClient) = _channels.Current();
             if (retryIndex != index)
             {
@@ -202,7 +204,7 @@ public sealed class GrpcCentralTransport : ICentralTransport
                     _logger.LogWarning(retryEx,
                         "Central control-plane call failed on both endpoints; surfacing as transient.");
                     var reported = retryEx is RpcException rpc && IsConnectFailure(rpc)
-                        && !retrySafe && !GrpcRetrySafety.WasNotDispatched(rpc)
+                        && !retrySafe && !GrpcRetrySafety.ProvablyNotApplied(rpc)
                             ? GrpcRetrySafety.OutcomeUnknown(rpc) : retryEx;
                     replyTo.Tell(new AkkaStatus.Failure(reported));
                     return;

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/GrpcRetrySafety.cs

@@ -1,3 +1,5 @@
+using System.Net.Http;
+using System.Net.Sockets;
 using Grpc.Core;
 
 namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
@@ -20,6 +22,67 @@ public static class GrpcRetrySafety
     public static bool WasNotDispatched(RpcException error) => error.StatusCode == StatusCode.Unavailable
         && error.Trailers.Any(e => e.Key == DispatchTrailer && e.Value == "not-started");
 
+    /// <summary>
+    /// Status detail Grpc.Net.Client sets when its load balancer could not connect a subchannel
+    /// and failed the call at PICK time — before any HTTP request for the call was created.
+    /// </summary>
+    internal const string SubchannelConnectFailureDetail = "Error connecting to subchannel.";
+
+    /// <summary>
+    /// True only when the client failed while OPENING a connection, so the request provably never
+    /// left this process (ScadaBridge#136 — the first command to a stopped node of a site pair).
+    /// Two shapes qualify, both <see cref="StatusCode.Unavailable"/> with a
+    /// <see cref="SocketException"/> in the debug-exception chain:
+    /// <list type="bullet">
+    /// <item>Grpc.Net.Client's subchannel connect failure (<see cref="SubchannelConnectFailureDetail"/>):
+    /// the balancer never handed the call a connection, whatever the socket error.</item>
+    /// <item>An <see cref="HttpRequestException"/> classified as a name-resolution or connection
+    /// error whose socket error can only arise before a connection exists (refused, host not
+    /// found, DNS try-again / no-data).</item>
+    /// </list>
+    /// Every other transport failure — a reset, GOAWAY, keepalive timeout or premature end on an
+    /// established connection — may have delivered the request and stays ambiguous.
+    /// </summary>
+    /// <param name="error">The RPC failure to inspect.</param>
+    /// <returns><c>true</c> when the request was never sent.</returns>
+    public static bool WasNeverSent(RpcException error)
+    {
+        if (error.StatusCode != StatusCode.Unavailable)
+        {
+            return false;
+        }
+
+        HttpRequestException? http = null;
+        SocketException? socket = null;
+        for (var e = error.Status.DebugException; e is not null; e = e.InnerException)
+        {
+            http ??= e as HttpRequestException;
+            socket ??= e as SocketException;
+        }
+
+        if (socket is null)
+        {
+            return false;
+        }
+
+        if (http is null)
+        {
+            return string.Equals(error.Status.Detail, SubchannelConnectFailureDetail, StringComparison.Ordinal);
+        }
+
+        return http.HttpRequestError is HttpRequestError.NameResolutionError or HttpRequestError.ConnectionError
+            && socket.SocketErrorCode is SocketError.ConnectionRefused or SocketError.HostNotFound
+                or SocketError.TryAgain or SocketError.NoData;
+    }
+
+    /// <summary>
+    /// True when a mutation may be replayed on the peer node: the server rejected it before
+    /// dispatch (<see cref="WasNotDispatched"/>), or the client never sent it (<see cref="WasNeverSent"/>).
+    /// </summary>
+    /// <param name="error">The RPC failure to inspect.</param>
+    /// <returns><c>true</c> when no node can have applied the request.</returns>
+    public static bool ProvablyNotApplied(RpcException error) => WasNotDispatched(error) || WasNeverSent(error);
+
     /// <summary>Reports an ambiguous mutation without inviting a blind replay.</summary>
     /// <param name="error">The original RPC failure whose trailers are preserved.</param>
     /// <returns>An <see cref="RpcException"/> with <c>Aborted</c> status marking the outcome as unknown.</returns>

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/SitePairChannelProvider.cs

@@ -35,7 +35,8 @@ public sealed class SiteChannelUnavailableException(string siteId)
 /// <remarks>
 /// <para>
 /// <b>Retry safety.</b> Unavailable can mean a lost response after application. Only explicitly
-/// retry-safe operations or a server-owned not-started trailer permit replay on the peer.
+/// retry-safe operations, a server-owned not-started trailer, or a connect-phase failure that
+/// provably never sent the request (<see cref="GrpcRetrySafety.WasNeverSent"/>) permit replay on the peer.
 /// Ambiguous mutations fail with Aborted / OUTCOME_UNKNOWN; the next independent call uses the
 /// peer. Deadlines are never replayed. Readiness rejection remains safe before actor dispatch.
 /// </para>
@@ -324,16 +325,20 @@ public sealed class SitePairChannelProvider : IDisposable
         catch (RpcException ex) when (ex.StatusCode == StatusCode.Unavailable)
         {
             if (secondary is not null) FlipTo(pair, toIsA: !primaryIsA);
-            if (!retrySafe && !GrpcRetrySafety.WasNotDispatched(ex))
+            // ScadaBridge#136: a connect-phase failure (refused / unresolvable — the stopped node of
+            // a pair) provably never sent the request, so a mutation fails over within this call
+            // just like an explicit pre-dispatch rejection. Anything that may have been delivered
+            // stays OUTCOME_UNKNOWN and is never replayed.
+            if (!retrySafe && !GrpcRetrySafety.ProvablyNotApplied(ex))
                 throw GrpcRetrySafety.OutcomeUnknown(ex);
             if (secondary is null) throw;
-            // Only reads/idempotent operations or an explicit pre-dispatch rejection may replay.
+            // Only reads/idempotent operations or a provably unapplied request may replay.
             _logger.LogWarning(
                 "Site {SiteId} node {FromNode} unavailable; failing over to node {ToNode}",
                 siteId, primaryIsA ? "A" : "B", primaryIsA ? "B" : "A");
             try { return await call(secondary, ct).ConfigureAwait(false); }
             catch (RpcException retryError) when (retryError.StatusCode == StatusCode.Unavailable
-                && !retrySafe && !GrpcRetrySafety.WasNotDispatched(retryError))
+                && !retrySafe && !GrpcRetrySafety.ProvablyNotApplied(retryError))
             {
                 throw GrpcRetrySafety.OutcomeUnknown(retryError);
             }