dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(#654): bounded fast retry while the known-cluster gate is unpublished

Browse Source 
CentralCommunicationActor: while the PSK provider is unresolved or the first
ClusterNode load failed, re-arm RefreshContacts every 2 s (max 15) until the
first publish; Information log on first publish; 'NEVER be published' Error
now only after the retry budget is exhausted. Corrected false comments.
Tests: late provider (Grpc+Dps), failed first load, retries stop, bounded budget;
ControlPlaneStarterOrderTests back to the default 60 s refresh.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 23:32:38 -04:00
parent ca17d2cf71
commit 9dc6390d50
4 changed files with 184 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs
+67 -13
View File
@@ -48,6 +48,18 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
{
private const string RefreshTimerKey = "mesh-contact-refresh";
private const string BootstrapRetryTimerKey = "mesh-contact-bootstrap-retry";
// While the known-cluster gate is unpublished (PSK provider unresolved, or the first
// ClusterNode load failed) a single-shot RefreshContacts is re-armed every
// DefaultBootstrapRetryInterval, at most MaxBootstrapRetries times (~30 s), stopping at the
// first publish. Constants, not config. Steady state costs nothing extra.
private static readonly TimeSpan DefaultBootstrapRetryInterval = TimeSpan.FromSeconds(2);
private const int MaxBootstrapRetries = 15;
private readonly TimeSpan _bootstrapRetryInterval;
private int _bootstrapRetries;
private int _refreshAttempts;
private bool _gatePublished;
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
private readonly MeshTransportOptions _options;
@@ -64,9 +76,11 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
// Memoize-on-first-use, NOT PreStart-only: PreStart runs on a dispatcher thread concurrently
// with the builder's remaining starters, so its resolve can lose a millisecond-wide race
// against AuditWriterActorKey's registration (starter #4) and Resolve() would swallow that
// into a PERMANENT silent null — the known-cluster gate never published, fail-open. These
// properties retry on first use instead; the first RefreshContacts arrives as a message,
// long after every starter finished, so a lost boot race self-heals there. Actor-thread
// into a silent null — the known-cluster gate unpublished, fail-open. These properties
// retry on first use, but the first RefreshContacts is sent from PreStart (TimeSpan.Zero) and
// can ALSO land inside the starter window (a fast DB load finishes in ms), so first-use is not
// enough: while the gate is unpublished a bounded fast retry (ScheduleBootstrapRetry) re-arms
// RefreshContacts every couple of seconds until it publishes. Actor-thread
// only, so ??= needs no synchronisation; both are container singletons, so a repeat resolve
// is a dictionary hit.
private IMeshPskProvider? PskProvider =>
@@ -118,6 +132,10 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
/// exist (the known-cluster gate) and to drop its cache (so a rotated key takes effect within
/// one refresh). Optional for the same reason as <paramref name="dialer"/>.
/// </param>
/// <param name="bootstrapRetryInterval">
/// Interval of the bounded fast retry run while the known-cluster gate is unpublished
/// (default 2 s, at most 15 attempts).
/// </param>
/// <returns>The props.</returns>
/// <remarks>
/// <para>
@@ -145,9 +163,11 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
MeshTransportOptions options,
Func<IActorRef?> coordinator,
Func<IMeshCommandDialer?>? dialer = null,
Func<IMeshPskProvider?>? pskProvider = null) =>
Func<IMeshPskProvider?>? pskProvider = null,
TimeSpan? bootstrapRetryInterval = null) =>
Akka.Actor.Props.Create(() =>
new CentralCommunicationActor(dbFactory, options, coordinator, dialer, pskProvider));
new CentralCommunicationActor(
dbFactory, options, coordinator, dialer, pskProvider, bootstrapRetryInterval));
/// <summary>Initializes a new instance of the <see cref="CentralCommunicationActor"/> class.</summary>
/// <param name="dbFactory">Factory for the config database.</param>
@@ -155,13 +175,16 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
/// <param name="coordinator">Lazily resolves the deploy-coordinator singleton proxy.</param>
/// <param name="dialer">Resolves the gRPC command dialer; invoked in <see cref="PreStart"/>.</param>
/// <param name="pskProvider">Resolves the PSK provider; invoked in <see cref="PreStart"/>.</param>
/// <param name="bootstrapRetryInterval">Fast-retry interval while the gate is unpublished; default 2 s.</param>
public CentralCommunicationActor(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
MeshTransportOptions options,
Func<IActorRef?> coordinator,
Func<IMeshCommandDialer?>? dialer = null,
Func<IMeshPskProvider?>? pskProvider = null)
Func<IMeshPskProvider?>? pskProvider = null,
TimeSpan? bootstrapRetryInterval = null)
{
_bootstrapRetryInterval = bootstrapRetryInterval ?? DefaultBootstrapRetryInterval;
_dbFactory = dbFactory;
_options = options;
_coordinator = coordinator;
@@ -172,7 +195,11 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
Receive<MeshCommand>(HandleMeshCommand);
Receive<ApplyAck>(HandleApplyAck);
Receive<RefreshContacts>(_ => LoadContactsFromDb());
Receive<RefreshContacts>(_ =>
{
_refreshAttempts++;
LoadContactsFromDb();
});
Receive<ContactsLoaded>(HandleContactsLoaded);
Receive<CommandDispatched>(HandleCommandDispatched);
@@ -185,9 +212,12 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
// widen the gate nor narrow it, and the previous set is the last thing central actually knew.
// The second line exists because that freeze is INVISIBLE otherwise, and it is fail-OPEN
// before the first successful load — a central that has never reached its ConfigDb refuses
// no cluster at all.
// no cluster at all. Until the first publish a bounded fast retry (above) heals it.
Receive<Status.Failure>(f =>
{
// Gate still unpublished (boot with ConfigDb briefly down): retry fast, bounded.
if (!_gatePublished)
ScheduleBootstrapRetry();
_log.Warning(
f.Cause,
"Failed to load ClusterNode contact points; the mesh contact set was NOT refreshed "
@@ -213,8 +243,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
// with the builder's remaining starters and loses a millisecond-wide race against
// AuditWriterActorKey's registration (starter #4) on essentially every admin boot (live
// gates: 1A 2/3 boots, 1B 4/4 boots). That is the ORDINARY case, not a malfunction — the
// memoized PskProvider/Dialer properties retry on first use, long after every starter
// finished, and the lost race self-heals there almost every time. Logging it as
// memoized PskProvider/Dialer properties retry on first use, and the bounded bootstrap
// retry re-arms RefreshContacts until the gate publishes (the first refresh itself can
// land inside the starter window). Logging it as
// Warning/Error twice a boot is what made a soak error-grep on a healthy rig impossible.
_dialer = Resolve(_dialerFactory, nameof(IMeshCommandDialer), firstUse: false);
_pskProvider = Resolve(_pskProviderFactory, nameof(IMeshPskProvider), firstUse: false);
@@ -250,6 +281,15 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
TimeSpan.FromSeconds(_options.ContactRefreshSeconds));
}
/// <summary>Re-arms a single-shot <see cref="RefreshContacts"/> while the gate is unpublished.</summary>
private void ScheduleBootstrapRetry()
{
if (_bootstrapRetries >= MaxBootstrapRetries)
return;
_bootstrapRetries++;
Timers.StartSingleTimer(BootstrapRetryTimerKey, new RefreshContacts(), _bootstrapRetryInterval);
}
/// <summary>
/// Invokes a dependency factory, converting a resolution failure into a logged absence.
/// </summary>
@@ -718,12 +758,17 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
// Gated on a factory having been supplied at all: a never-supplied factory is a
// deliberate wiring/test choice Resolve() already stayed silent about (see its
// remarks), not a resolve failure, so it earns no Error here either.
if (_pskProviderFactory is not null && !_pskProviderAbsenceReported)
if (_pskProviderFactory is not null && !_gatePublished)
ScheduleBootstrapRetry();
// Error only once the bounded fast retry is exhausted, so a healed boot race stays quiet.
if (_pskProviderFactory is not null && !_pskProviderAbsenceReported
&& _bootstrapRetries >= MaxBootstrapRetries)
{
_pskProviderAbsenceReported = true;
_log.Error(
"No IMeshPskProvider is resolvable for this actor, even after a first-use retry; "
+ "the known-cluster gate will NEVER be published and cached per-cluster keys "
"No IMeshPskProvider is resolvable for this actor, even after a first-use retry and "
+ "the bounded bootstrap retries; the known-cluster gate will NEVER be published and cached per-cluster keys "
+ "will NEVER be dropped. Inbound mesh acks stay exposed to the "
+ "pre-authentication secret-store cost the gate removes, and a rotated key "
+ "needs a central restart. Check that the ControlPlane registration wires up "
@@ -735,6 +780,15 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
pskProvider.SetKnownClusters(msg.ClusterIds);
pskProvider.InvalidateAll();
if (!_gatePublished)
{
_gatePublished = true;
Timers.Cancel(BootstrapRetryTimerKey);
_log.Info(
"Known-cluster gate published ({Count} clusters) after {Attempts} refresh attempt(s)",
msg.ClusterIds.Count, _refreshAttempts);
}
}
/// <summary>Internal tick asking for a contact-point refresh.</summary>