fix(communication): malformed site gRPC address keeps the old channel, never aborts the refresh tick or restarts CentralCommunicationActor (review 2026-10-21 03/R-1)

2026-10-22 06:14:34 -04:00
parent 0fd9bcbce6
commit b0a97a4b67
5 changed files with 261 additions and 8 deletions
@@ -392,8 +392,19 @@ public class CentralCommunicationActor : ReceiveActor
    private void HandleSiteAddressCacheLoaded(SiteAddressCacheLoaded msg)
    {
        // Per-site transport resource reconciliation (build/drop gRPC channel
-        // pairs). Runs on the actor thread each refresh tick.
-        _transport.ReconcileSites(msg);
+        // pairs). Runs on the actor thread each refresh tick. Guarded (review 2026-10-21 03/R-1):
+        // a throw here would restart this actor, whose PreStart re-reads the Sites table at
+        // TimeSpan.Zero — a restart loop for as long as a bad row persists — and would skip the
+        // aggregator prune below.
+        try
+        {
+            _transport.ReconcileSites(msg);
+        }
+        catch (Exception ex)
+        {
+            _log.Error(ex, "Reconciling site gRPC channels from the refreshed address cache failed; "
+                + "existing channels are kept until the next refresh");
+        }

        // Self-healing eviction: a site deleted from configuration would otherwise
        // linger in the aggregator as a permanently-offline tile (and live KPI
@@ -245,18 +245,38 @@ public sealed class GrpcSiteTransport : ISiteCommandTransport

        var desired = cache.GrpcContacts;

+        // Each site is reconciled independently (review 2026-10-21 03/R-1): one bad row must not
+        // abort the rest of the tick, nor throw into CentralCommunicationActor's Receive (whose
+        // restart re-reads the Sites table immediately — a restart loop on a persistent bad row).
        // Drop sites that lost their gRPC endpoints (or were deleted).
        foreach (var removed in _knownSites.Where(s => !desired.ContainsKey(s)).ToList())
        {
-            _channels.RemoveSite(removed);
-            _knownSites.Remove(removed);
+            try
+            {
+                _channels.RemoveSite(removed);
+                _knownSites.Remove(removed);
+            }
+            catch (Exception ex) when (ex is not ObjectDisposedException)
+            {
+                _logger.LogError(ex, "Failed to remove the gRPC channel pair for site {SiteId}", removed);
+            }
        }

        // Create/refresh the rest.
        foreach (var (siteId, endpoints) in desired)
        {
-            _channels.UpdateSite(siteId, endpoints.NodeA, endpoints.NodeB);
-            _knownSites.Add(siteId);
+            try
+            {
+                _channels.UpdateSite(siteId, endpoints.NodeA, endpoints.NodeB);
+                _knownSites.Add(siteId);
+            }
+            catch (Exception ex) when (ex is not ObjectDisposedException)
+            {
+                _logger.LogError(ex,
+                    "Failed to reconcile the gRPC channel pair for site {SiteId} (nodeA={NodeA}, nodeB={NodeB}); "
+                    + "its previous channels, if any, are kept",
+                    siteId, endpoints.NodeA ?? "none", endpoints.NodeB ?? "none");
+            }
        }
    }
 }
@@ -3,6 +3,7 @@ using Grpc.Core;
 using Grpc.Net.Client;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.Commons.Validation;

 namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;

@@ -129,6 +130,22 @@ public sealed class SitePairChannelProvider : IDisposable
            {
                var changedA = !string.Equals(pair.NodeAEndpoint, nodeAEndpoint, StringComparison.Ordinal);
                var changedB = !string.Equals(pair.NodeBEndpoint, nodeBEndpoint, StringComparison.Ordinal);
+
+                // Build every replacement BEFORE parking anything (review 2026-10-21 03/R-1): a
+                // malformed address used to throw out of BuildChannel after the old channel was
+                // already parked, leaving pair.ChannelA pointing at a channel the graveyard would
+                // dispose — every later call then failed with ObjectDisposedException (not
+                // Unavailable, so no failover) — and the throw escaped into the refresh actor.
+                // A side whose new endpoint cannot be built keeps its old channel AND old endpoint.
+                GrpcChannel? newA = null, newB = null;
+                if (changedA && !TryBuildReplacement(pair, siteId, nodeAEndpoint, isA: true, out newA))
+                {
+                    changedA = false;
+                }
+                if (changedB && !TryBuildReplacement(pair, siteId, nodeBEndpoint, isA: false, out newB))
+                {
+                    changedB = false;
+                }
                if (!changedA && !changedB)
                {
                    return;
@@ -137,13 +154,13 @@ public sealed class SitePairChannelProvider : IDisposable
                if (changedA)
                {
                    Park(pair.ChannelA);
-                    pair.ChannelA = string.IsNullOrWhiteSpace(nodeAEndpoint) ? null : BuildChannel(nodeAEndpoint, siteId);
+                    pair.ChannelA = newA;
                    pair.NodeAEndpoint = nodeAEndpoint;
                }
                if (changedB)
                {
                    Park(pair.ChannelB);
-                    pair.ChannelB = string.IsNullOrWhiteSpace(nodeBEndpoint) ? null : BuildChannel(nodeBEndpoint, siteId);
+                    pair.ChannelB = newB;
                    pair.NodeBEndpoint = nodeBEndpoint;
                }

@@ -158,6 +175,62 @@ public sealed class SitePairChannelProvider : IDisposable
        }
    }

+    /// <summary>
+    /// Builds the replacement channel for one side of a pair whose endpoint changed. A null/blank
+    /// endpoint is a valid "unconfigured" (null channel). A malformed endpoint — validated with
+    /// <see cref="GrpcEndpointRules"/>, and any channel-construction throw besides — is rejected:
+    /// logged at Error naming the site and column (once per distinct rejected value, so the 60 s
+    /// refresh does not repeat it every tick) and the caller keeps the side's existing channel.
+    /// </summary>
+    /// <param name="pair">The site pair (caller holds <c>pair.Gate</c>).</param>
+    /// <param name="siteId">Site identifier, for the log.</param>
+    /// <param name="endpoint">The new endpoint for this side.</param>
+    /// <param name="isA">True for NodeA (<c>GrpcNodeAAddress</c>), false for NodeB.</param>
+    /// <param name="channel">The built channel, or null for an unconfigured side.</param>
+    /// <returns>True when the replacement may be applied; false to keep the existing side.</returns>
+    private bool TryBuildReplacement(SitePair pair, string siteId, string? endpoint, bool isA, out GrpcChannel? channel)
+    {
+        channel = null;
+        if (string.IsNullOrWhiteSpace(endpoint))
+        {
+            SetRejected(pair, isA, null);
+            return true;
+        }
+
+        var field = isA ? "GrpcNodeAAddress" : "GrpcNodeBAddress";
+        Exception? buildError = null;
+        if (GrpcEndpointRules.TryValidate(endpoint, allowEmpty: false, out var error))
+        {
+            try
+            {
+                channel = BuildChannel(endpoint, siteId);
+                SetRejected(pair, isA, null);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                buildError = ex;
+                error = ex.Message;
+            }
+        }
+
+        if (!string.Equals(isA ? pair.RejectedAEndpoint : pair.RejectedBEndpoint, endpoint, StringComparison.Ordinal))
+        {
+            _logger.LogError(buildError,
+                "Site {SiteId} {Field} '{Endpoint}' is not a usable gRPC endpoint ({Error}); keeping the "
+                + "previous channel ({Previous}) for that node until the address is corrected",
+                siteId, field, endpoint, error, (isA ? pair.NodeAEndpoint : pair.NodeBEndpoint) ?? "none");
+            SetRejected(pair, isA, endpoint);
+        }
+        return false;
+    }
+
+    private static void SetRejected(SitePair pair, bool isA, string? endpoint)
+    {
+        if (isA) pair.RejectedAEndpoint = endpoint;
+        else pair.RejectedBEndpoint = endpoint;
+    }
+
    /// <summary>
    /// Parks a removed site's channels for deferred disposal (see <see cref="ChannelDisposalGrace"/>)
    /// and invalidates its cached preshared key.
@@ -504,6 +577,10 @@ public sealed class SitePairChannelProvider : IDisposable
        public readonly object Gate = new();
        public string? NodeAEndpoint;
        public string? NodeBEndpoint;
+
+        /// <summary>Last malformed NodeA/NodeB endpoint rejected by <c>UpdateSite</c> (log de-duplication only).</summary>
+        public string? RejectedAEndpoint;
+        public string? RejectedBEndpoint;
        public GrpcChannel? ChannelA;
        public GrpcChannel? ChannelB;