dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(communication): site-address refresh gets an overlap guard, per-load deadline and monotonic sequence (review 2026-10-21 03/M-1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 06:16:15 -04:00
parent b0a97a4b67
commit 3ed71167b3
4 changed files with 244 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/Actors/CentralCommunicationActor.cs
+123 -25
View File
@@ -3,6 +3,7 @@ using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
@@ -58,6 +59,22 @@ public class CentralCommunicationActor : ReceiveActor
/// </summary>
private readonly CancellationTokenSource _lifecycleCts = new();
/// <summary>
/// Process-wide monotonic stamp for <see cref="SiteAddressCacheLoaded"/> (review 2026-10-21
/// 03/M-1). Static so it stays monotonic across actor restarts: a late result piped from a
/// previous incarnation's load can never outrank a newer one.
/// </summary>
private static long s_loadSequence;
/// <summary>Sequence of the DB load currently in flight; 0 when none (the overlap guard).</summary>
private long _inFlightLoadSequence;
/// <summary>A refresh requested while a load was in flight; re-run once the load completes.</summary>
private bool _refreshPending;
/// <summary>Sequence of the last result handed to the transport; older results are dropped.</summary>
private long _lastAppliedLoadSequence;
/// <summary>
/// Proxy <see cref="IActorRef"/> for the central NotificationOutboxActor cluster singleton.
/// Set via <see cref="RegisterNotificationOutbox"/> — the Host creates the singleton proxy
@@ -103,10 +120,11 @@ public class CentralCommunicationActor : ReceiveActor
// Periodic refresh trigger
Receive<RefreshSiteAddresses>(_ => LoadSiteAddressesFromDb());
// A faulted LoadSiteAddressesFromDb task is piped here as a
// Status.Failure. Without this handler the failure was an unhandled message
// (debug-level only) and the refresh failed silently — operators could not
// distinguish "no sites configured" from "database is down". Log at Warning.
// A faulted (or deadline-cancelled) LoadSiteAddressesFromDb task. Without this the
// failure was an unhandled message (debug-level only) and the refresh failed silently —
// operators could not distinguish "no sites configured" from "database is down". Log at
// Warning and release the overlap guard.
Receive<SiteAddressLoadFailed>(HandleSiteAddressLoadFailed);
Receive<Status.Failure>(failure =>
_log.Warning(failure.Cause,
"Failed to load site addresses from the database; the per-site gRPC channel "
@@ -348,49 +366,118 @@ public class CentralCommunicationActor : ReceiveActor
private void LoadSiteAddressesFromDb()
{
// Overlap guard (review 2026-10-21 03/M-1): a wedged DB used to stack one Task.Run + DI
// scope + DbContext per 60 s tick, and their late completions could land out of order.
// A tick (or an admin-change refresh) that arrives mid-load is coalesced into ONE re-run
// once the current load finishes, so an on-demand refresh is never lost.
if (_inFlightLoadSequence != 0)
{
_refreshPending = true;
return;
}
var self = Self;
// Pass the actor's lifecycle CT into the repository
// call so a hung database query is cancelled when the actor stops
// rather than leaving the piped task to accumulate. Captured locally
// because the lifecycle CTS may have been disposed by PostStop on a
// racing late tick; treat that as "actor gone, give up".
CancellationToken ct;
CancellationToken lifecycle;
try
{
ct = _lifecycleCts.Token;
lifecycle = _lifecycleCts.Token;
}
catch (ObjectDisposedException)
{
return;
}
var sequence = Interlocked.Increment(ref s_loadSequence);
_inFlightLoadSequence = sequence;
_refreshPending = false;
// Per-load deadline: a hung read is abandoned instead of pinning the overlap guard.
var deadline = _serviceProvider.GetService<IOptions<CommunicationOptions>>()?.Value.QueryTimeout
?? TimeSpan.FromSeconds(30);
Task.Run(async () =>
{
using var scope = _serviceProvider.CreateScope();
var repo = scope.ServiceProvider.GetRequiredService<ISiteRepository>();
var sites = await repo.GetAllSitesAsync(ct).ConfigureAwait(false);
// gRPC-endpoint cache (the GrpcNodeA/BAddress columns, NOT the legacy Akka
// NodeA/BAddress ones, which no longer drive cross-cluster dialling).
var grpcContacts = new Dictionary<string, SiteGrpcEndpoints>();
foreach (var site in sites)
using var loadCts = CancellationTokenSource.CreateLinkedTokenSource(lifecycle);
loadCts.CancelAfter(deadline);
try
{
var grpcA = string.IsNullOrWhiteSpace(site.GrpcNodeAAddress) ? null : site.GrpcNodeAAddress;
var grpcB = string.IsNullOrWhiteSpace(site.GrpcNodeBAddress) ? null : site.GrpcNodeBAddress;
if (grpcA is not null || grpcB is not null)
grpcContacts[site.SiteIdentifier] = new SiteGrpcEndpoints(grpcA, grpcB);
}
using var scope = _serviceProvider.CreateScope();
var repo = scope.ServiceProvider.GetRequiredService<ISiteRepository>();
var sites = await repo.GetAllSitesAsync(loadCts.Token).ConfigureAwait(false);
// Carry the full configured-site identifier set (not just the
// gRPC-address-bearing subset) so the aggregator prunes only
// genuinely-deleted sites and never an addressless-but-configured one.
var knownSiteIds = sites.Select(s => s.SiteIdentifier).ToList();
return new SiteAddressCacheLoaded(knownSiteIds, grpcContacts);
// gRPC-endpoint cache (the GrpcNodeA/BAddress columns, NOT the legacy Akka
// NodeA/BAddress ones, which no longer drive cross-cluster dialling).
var grpcContacts = new Dictionary<string, SiteGrpcEndpoints>();
foreach (var site in sites)
{
var grpcA = string.IsNullOrWhiteSpace(site.GrpcNodeAAddress) ? null : site.GrpcNodeAAddress;
var grpcB = string.IsNullOrWhiteSpace(site.GrpcNodeBAddress) ? null : site.GrpcNodeBAddress;
if (grpcA is not null || grpcB is not null)
grpcContacts[site.SiteIdentifier] = new SiteGrpcEndpoints(grpcA, grpcB);
}
// Carry the full configured-site identifier set (not just the
// gRPC-address-bearing subset) so the aggregator prunes only
// genuinely-deleted sites and never an addressless-but-configured one.
var knownSiteIds = sites.Select(s => s.SiteIdentifier).ToList();
return (object)new SiteAddressCacheLoaded(knownSiteIds, grpcContacts, sequence);
}
catch (Exception ex)
{
var timedOut = ex is OperationCanceledException && !lifecycle.IsCancellationRequested;
return new SiteAddressLoadFailed(sequence, ex, timedOut ? deadline : null);
}
}).PipeTo(self);
}
private void CompleteLoad(long sequence)
{
if (sequence != _inFlightLoadSequence)
{
return; // a previous incarnation's / superseded load — the guard belongs to another one
}
_inFlightLoadSequence = 0;
if (_refreshPending)
{
LoadSiteAddressesFromDb();
}
}
private void HandleSiteAddressLoadFailed(SiteAddressLoadFailed failed)
{
if (failed.TimedOutAfter is { } after)
{
_log.Warning(
"Loading site addresses from the database did not complete within {0}; the per-site "
+ "gRPC channel cache was not refreshed and may be stale or empty", after);
}
else
{
_log.Warning(failed.Cause,
"Failed to load site addresses from the database; the per-site gRPC channel "
+ "cache was not refreshed and may be stale or empty");
}
CompleteLoad(failed.Sequence);
}
private void HandleSiteAddressCacheLoaded(SiteAddressCacheLoaded msg)
{
CompleteLoad(msg.Sequence);
// Never let an older read overwrite a newer one (review 2026-10-21 03/M-1): applying
// stale addresses would park freshly built channels and rebuild from the old endpoints.
if (msg.Sequence <= _lastAppliedLoadSequence)
{
_log.Debug("Dropping stale site-address load {0} (last applied {1})",
msg.Sequence, _lastAppliedLoadSequence);
return;
}
_lastAppliedLoadSequence = msg.Sequence;
// Per-site transport resource reconciliation (build/drop gRPC channel
// pairs). Runs on the actor thread each refresh tick. Guarded (review 2026-10-21 03/R-1):
// a throw here would restart this actor, whose PreStart re-reads the Sites table at
@@ -490,9 +577,20 @@ public record RefreshSiteAddresses;
/// </summary>
/// <param name="KnownSiteIds">Every configured site id, address-bearing or not, for aggregator pruning.</param>
/// <param name="GrpcContacts">gRPC endpoint pairs per site (from GrpcNodeA/GrpcNodeBAddress).</param>
/// <param name="Sequence">
/// Monotonic load stamp; the actor drops a result whose sequence is not newer than the last one it
/// applied, so an out-of-order completion never re-applies stale addresses.
/// </param>
public sealed record SiteAddressCacheLoaded(
IReadOnlyCollection<string> KnownSiteIds,
IReadOnlyDictionary<string, SiteGrpcEndpoints> GrpcContacts);
IReadOnlyDictionary<string, SiteGrpcEndpoints> GrpcContacts,
long Sequence);
/// <summary>A failed (or deadline-cancelled) site-address DB load, piped back to release the overlap guard.</summary>
/// <param name="Sequence">The failed load's sequence stamp.</param>
/// <param name="Cause">The failure.</param>
/// <param name="TimedOutAfter">The per-load deadline when the load was cancelled by it; otherwise null.</param>
internal sealed record SiteAddressLoadFailed(long Sequence, Exception Cause, TimeSpan? TimedOutAfter);
/// <summary>
/// A site's gRPC node-pair endpoints, as loaded from <c>Site.GrpcNodeAAddress</c>/