fix(communication): fail a site command over within the call when the stopped node never received it (#136)

The first mutation (instance deploy) to a site pair whose sticky node was down returned OUTCOME_UNKNOWN: SitePairChannelProvider.ExecuteAsync treated every Unavailable on a non-retry-safe command as possibly-applied unless the server sent the not-dispatched trailer, flipped stickiness, and threw. The retry then succeeded on the live peer. A connect-phase failure provably never sent the request, so it is now treated like the pre-dispatch rejection: GrpcRetrySafety.WasNeverSent recognises Grpc.Net.Client's subchannel connect failure ("Error connecting to subchannel." with a SocketException cause) and an HttpRequestException name-resolution / connection error with a connect-only socket error. Anything that may have been delivered (reset, premature end, GOAWAY, server Unavailable) stays OUTCOME_UNKNOWN and is not replayed. The site->central reconcile path uses the same ProvablyNotApplied rule. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 00:22:54 -05:00
parent d7fa31f0b8
commit fd80aeab57
5 changed files with 244 additions and 9 deletions
@@ -0,0 +1,73 @@
+using System.Net.Http;
+using System.Net.Sockets;
+using Grpc.Core;
+using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
+
+namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
+
+/// <summary>
+/// ScadaBridge#136: <see cref="GrpcRetrySafety.WasNeverSent"/> admits only connect-phase failures
+/// (the request provably never left the client) and rejects every shape that may have delivered it.
+/// </summary>
+public sealed class GrpcRetrySafetyNeverSentTests
+{
+    private static RpcException Rpc(StatusCode code, string detail, Exception? debug)
+        => new(new Status(code, detail, debug));
+
+    private static SocketException Sock(SocketError error) => new((int)error);
+
+    /// <summary>The balancer's subchannel connect failure is pre-send, whatever the socket error.</summary>
+    /// <param name="error">The socket error the subchannel connect saw.</param>
+    [Theory]
+    [InlineData(SocketError.ConnectionRefused)]
+    [InlineData(SocketError.HostNotFound)]
+    [InlineData(SocketError.TryAgain)]
+    [InlineData(SocketError.HostUnreachable)]
+    [InlineData(SocketError.TimedOut)]
+    public void SubchannelConnectFailure_IsNeverSent(SocketError error)
+    {
+        var ex = Rpc(StatusCode.Unavailable, GrpcRetrySafety.SubchannelConnectFailureDetail, Sock(error));
+
+        Assert.True(GrpcRetrySafety.WasNeverSent(ex));
+        Assert.True(GrpcRetrySafety.ProvablyNotApplied(ex));
+    }
+
+    /// <summary>An HttpRequestException connect error with a connect-only socket error is pre-send.</summary>
+    /// <param name="kind">The HTTP request error classification.</param>
+    /// <param name="error">The socket error.</param>
+    [Theory]
+    [InlineData(HttpRequestError.ConnectionError, SocketError.ConnectionRefused)]
+    [InlineData(HttpRequestError.NameResolutionError, SocketError.HostNotFound)]
+    [InlineData(HttpRequestError.NameResolutionError, SocketError.TryAgain)]
+    public void HttpConnectError_WithConnectOnlySocketError_IsNeverSent(HttpRequestError kind, SocketError error)
+    {
+        var ex = Rpc(StatusCode.Unavailable, "Error starting gRPC call.",
+            new HttpRequestException(kind, "connect failed", Sock(error)));
+
+        Assert.True(GrpcRetrySafety.WasNeverSent(ex));
+    }
+
+    /// <summary>Shapes that may follow delivery stay ambiguous.</summary>
+    [Fact]
+    public void FailuresThatMayFollowDelivery_AreNotNeverSent()
+    {
+        // Premature end / reset on an established connection.
+        Assert.False(GrpcRetrySafety.WasNeverSent(Rpc(StatusCode.Unavailable, "Error starting gRPC call.",
+            new HttpRequestException(HttpRequestError.ResponseEnded, "ended",
+                new IOException("reset", Sock(SocketError.ConnectionReset))))));
+        // A ConnectionError classification with a post-connect socket error.
+        Assert.False(GrpcRetrySafety.WasNeverSent(Rpc(StatusCode.Unavailable, "Error starting gRPC call.",
+            new HttpRequestException(HttpRequestError.ConnectionError, "x", Sock(SocketError.ConnectionReset)))));
+        // A server-sent Unavailable (no transport exception at all).
+        Assert.False(GrpcRetrySafety.WasNeverSent(Rpc(StatusCode.Unavailable, "modelled fault", null)));
+        // A bare socket error outside the subchannel-pick path.
+        Assert.False(GrpcRetrySafety.WasNeverSent(Rpc(StatusCode.Unavailable, "Error starting gRPC call.",
+            Sock(SocketError.ConnectionRefused))));
+        // Subchannel detail without a socket cause.
+        Assert.False(GrpcRetrySafety.WasNeverSent(Rpc(StatusCode.Unavailable,
+            GrpcRetrySafety.SubchannelConnectFailureDetail, new InvalidOperationException())));
+        // Not Unavailable.
+        Assert.False(GrpcRetrySafety.WasNeverSent(Rpc(StatusCode.DeadlineExceeded,
+            GrpcRetrySafety.SubchannelConnectFailureDetail, Sock(SocketError.ConnectionRefused))));
+    }
+}