dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(communication): command-plane failback probes readiness with a real RPC; backoff resets only after a real call succeeds (review 2026-10-21 03/P-2)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 06:26:13 -04:00
parent 017308a397
commit c5add11546
3 changed files with 255 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/SitePairChannelProvider.cs
+90 -14
View File
@@ -3,6 +3,7 @@ using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Validation;
namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
@@ -27,8 +28,9 @@ public sealed class SiteChannelUnavailableException(string siteId)
/// Central holds one <see cref="GrpcChannel"/> per site node (NodeA/NodeB) rather than one channel
/// with re-dial logic; NodeA is the preferred node (config order). Every call goes to the current
/// sticky channel; on connect failure / <see cref="StatusCode.Unavailable"/> / readiness rejection
/// it flips to the other node and stays there. A background probe returns to the preferred node
/// once it answers again. Reconnect probing backs off 1 s → doubling → 60 s cap.
/// it flips to the other node and stays there. A background readiness probe (a real read-only RPC)
/// returns to the preferred node once it answers again; the backoff (1 s → doubling → 60 s cap) is
/// reset only after a real call on the preferred node succeeds.
/// </summary>
/// <remarks>
/// <para>
@@ -87,8 +89,9 @@ public sealed class SitePairChannelProvider : IDisposable
/// a keepalive-configured <see cref="SocketsHttpHandler"/>.
/// </param>
/// <param name="reachabilityProbe">
/// Test seam deciding whether a preferred node is reachable during a failback sweep. Null in
/// production, where <see cref="GrpcChannel.ConnectAsync"/> is used.
/// Test seam deciding whether a preferred node is READY during a failback sweep. Null in
/// production, where <see cref="ReadinessProbeAsync"/> issues a real, read-only
/// <c>ExecuteLifecycle</c> call (review 2026-10-21 03/P-2).
/// </param>
public SitePairChannelProvider(
ISitePskProvider pskProvider,
@@ -105,7 +108,7 @@ public sealed class SitePairChannelProvider : IDisposable
_options = options.Value;
_logger = logger;
_handlerFactory = handlerFactory;
_reachabilityProbe = reachabilityProbe ?? DefaultReachabilityProbe;
_reachabilityProbe = reachabilityProbe ?? ReadinessProbeAsync;
}
/// <summary>
@@ -306,7 +309,9 @@ public sealed class SitePairChannelProvider : IDisposable
try
{
return await call(primary, ct).ConfigureAwait(false);
var result = await call(primary, ct).ConfigureAwait(false);
if (primaryIsA) ConfirmFailback(pair);
return result;
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.Unavailable)
{
@@ -362,9 +367,14 @@ public sealed class SitePairChannelProvider : IDisposable
{
if (reachable)
{
// Return to A on PROBATION (review 2026-10-21 03/P-2): the backoff is kept, not
// reset, until a real call on A succeeds (ConfirmFailback). If A rejects the next
// call anyway, FlipTo doubles the backoff instead of restarting it at 1 s, so a
// node that answers the probe but not real traffic is retried ever less often.
pair.CurrentIsA = true;
pair.ResetFailback();
_logger.LogInformation("Site {SiteId} preferred node A reachable again; failing back", siteId);
pair.FailbackOnProbation = true;
pair.NextProbeUtc = DateTime.MaxValue;
_logger.LogInformation("Site {SiteId} preferred node A ready again; failing back", siteId);
return true;
}
@@ -494,6 +504,30 @@ public sealed class SitePairChannelProvider : IDisposable
/// <summary>Test/diagnostic accessor: how many replaced channels are awaiting deferred disposal.</summary>
internal int ParkedChannelCount => _graveyard.Count;
/// <summary>A real call on the preferred node succeeded: a probationary failback is confirmed and the backoff reset.</summary>
/// <param name="pair">The site pair.</param>
private static void ConfirmFailback(SitePair pair)
{
if (!pair.FailbackOnProbation) return;
lock (pair.Gate)
{
if (pair.FailbackOnProbation && pair.CurrentIsA)
{
pair.FailbackOnProbation = false;
pair.ResetFailback();
}
}
}
/// <summary>Test/diagnostic accessor: the site's current failback backoff (null when the site is unknown).</summary>
/// <param name="siteId">Site identifier.</param>
/// <returns>The backoff the next failback probe is (or would be) scheduled with.</returns>
internal TimeSpan? GetFailbackBackoff(string siteId)
{
if (!_sites.TryGetValue(siteId, out var pair)) return null;
lock (pair.Gate) return pair.Backoff;
}
private void FlipTo(SitePair pair, bool toIsA)
{
lock (pair.Gate)
@@ -501,11 +535,22 @@ public sealed class SitePairChannelProvider : IDisposable
pair.CurrentIsA = toIsA;
if (!toIsA)
{
// Failed over off the preferred node → schedule the first failback probe.
pair.ScheduleFailback(InitialBackoff);
if (pair.FailbackOnProbation)
{
// A failed us again right after a probe-driven failback: it is not really
// ready yet — grow the backoff rather than restarting it.
pair.FailbackOnProbation = false;
pair.BumpFailbackBackoff();
}
else
{
// Failed over off the preferred node → schedule the first failback probe.
pair.ScheduleFailback(InitialBackoff);
}
}
else
{
pair.FailbackOnProbation = false;
pair.ResetFailback();
}
}
@@ -529,15 +574,37 @@ public sealed class SitePairChannelProvider : IDisposable
return GrpcChannel.ForAddress(endpoint, channelOptions);
}
private static async Task<bool> DefaultReachabilityProbe(GrpcChannel channel, CancellationToken ct)
/// <summary>Sentinel instance name the readiness probe queries; never a real instance.</summary>
internal const string ReadinessProbeInstance = "__scadabridge-failback-probe__";
/// <summary>
/// Production failback probe (review 2026-10-21 03/P-2): a real, read-only
/// <c>ExecuteLifecycle(DeploymentStateQuery)</c> for a sentinel instance, NOT a transport
/// connect. <c>SiteCommandGrpcService</c> refuses every RPC with a not-dispatched
/// <see cref="StatusCode.Unavailable"/> until the site has finished bring-up, so a node that is
/// Up but still warming (minutes on a large site) now reads as NOT ready — a TCP/HTTP2 connect
/// succeeded there and failed every command back to it each sweep. The query reads one
/// (absent) row from the site's deployed-config store; it is idempotent and side-effect free.
/// Any reply is "ready"; any failure, including the readiness rejection, is "not ready".
/// </summary>
/// <param name="channel">The preferred node's channel.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>True when the node answered a real command.</returns>
internal static async Task<bool> ReadinessProbeAsync(GrpcChannel channel, CancellationToken ct)
{
try
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
timeout.CancelAfter(TimeSpan.FromSeconds(5));
await channel.ConnectAsync(timeout.Token).ConfigureAwait(false);
var request = SiteCommandDtoMapper.ToLifecycleRequest(new DeploymentStateQueryRequest(
"failback-probe-" + Guid.NewGuid().ToString("N"), ReadinessProbeInstance, DateTimeOffset.UtcNow));
var client = new SiteCommandService.SiteCommandServiceClient(channel);
await client.ExecuteLifecycleAsync(
request, deadline: DateTime.UtcNow.AddSeconds(5), cancellationToken: ct).ResponseAsync.ConfigureAwait(false);
return true;
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch
{
return false;
@@ -587,6 +654,15 @@ public sealed class SitePairChannelProvider : IDisposable
/// <summary>Sticky current node; NodeA is preferred. True = pointing at NodeA.</summary>
public bool CurrentIsA = true;
/// <summary>
/// True between a probe-driven failback and the first successful real call on NodeA; while
/// set, the backoff is kept so a renewed failure doubles it.
/// </summary>
public bool FailbackOnProbation;
/// <summary>The current failback backoff.</summary>
public TimeSpan Backoff => _backoff;
/// <summary>Next time a failback probe is due (while failed over). MaxValue = not scheduled.</summary>
public DateTime NextProbeUtc = DateTime.MaxValue;