Merge review R1 fixes: removed driver's resilience tracker released; SDK timed-unshelve no-op (engine owns expiry); commented re-ack stored as AddComment; native-ack Primary verdict published from start; drain gate SharesRowsWithPeer; malformed cache bytes don't adopt the revision; HostAbortedException passthrough; ClusterRoleInfo seed filtered

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 07:47:57 -04:00
28 changed files with 691 additions and 136 deletions
@@ -76,7 +76,7 @@ public sealed class ClusterRoleInfo : IClusterRoleInfo, IDisposable
}
}
SeedFromCurrentState();
SeedFrom(_cluster.State);
_subscriber = system.ActorOf(Props.Create(() => new SubscriberActor(this)), "clusterroleinfo-subscriber");
}
@@ -121,12 +121,19 @@ public sealed class ClusterRoleInfo : IClusterRoleInfo, IDisposable
/// <summary>Occurs when the leader for a role changes.</summary>
public event EventHandler<RoleLeaderChangedEventArgs>? RoleLeaderChanged;
private void SeedFromCurrentState()
/// <summary>Seeds the member/leader maps from a cluster-state snapshot (the constructor passes the live
/// <c>Cluster.State</c>; exposed for tests).</summary>
/// <param name="snapshot">The cluster-state snapshot.</param>
internal void SeedFrom(ClusterEvent.CurrentClusterState snapshot)
{
var snapshot = _cluster.State;
lock (_lock)
{
foreach (var member in snapshot.Members)
// Cluster-002 (review R1 m6): seed only USABLE members. A member already Leaving / Exiting / Down
// (or still Joining) at seed time is not one, and a departing member never fires a later
// MemberLeft / MemberExited / MemberDowned that HandleMemberEvent would drop it on — it would
// linger until MemberRemoved, the very delay the member-event handling exists to avoid. A
// Joining member is added by its MemberUp.
foreach (var member in snapshot.Members.Where(m => m.Status is MemberStatus.Up or MemberStatus.WeaklyUp))
foreach (var role in member.Roles)
{
if (!_membersByRole.TryGetValue(role, out var set))
@@ -28,6 +28,16 @@ public interface IDriverCapabilityInvokerFactory
/// </param>
/// <returns>A per-instance <see cref="IDriverCapabilityInvoker"/>.</returns>
IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null);
/// <summary>
/// Release every piece of shared resilience state held for <paramref name="driverInstanceId"/> — called
/// by the host when the instance leaves the deployed plan (#609). A concrete factory drops the
/// instance's cached pipelines and its status-tracker snapshots, so the periodic resilience publisher
/// stops re-emitting (and re-priming new telemetry subscribers with) a decommissioned driver's last
/// breaker/failure counters. Default: no-op (a factory that holds no state).
/// </summary>
/// <param name="driverInstanceId">The driver instance that was removed.</param>
void Release(string driverInstanceId) { }
}
/// <summary>
@@ -105,7 +105,17 @@ public static class Part9StateMachine
throw new ArgumentException("User identity required for audit.", nameof(user));
if (current.Acked == AlarmAckedState.Acknowledged)
return TransitionResult.NoOp(current, "already acknowledged");
{
// A COMMENTED re-ack carries a real comment even though nothing is acknowledged (Gitea #597 review
// m1): the OPC UA node reports it — the SDK writes the comment on a re-ack — so the system of
// record must hold it too, or the event advertises a comment that exists only on the node that
// served the call (never persisted, never replicated to the pair peer). Recorded exactly like
// AddComment; the ack fields (LastAck*) stay those of the real acknowledge. An uncommented re-ack
// is still a no-op.
return string.IsNullOrWhiteSpace(comment)
? TransitionResult.NoOp(current, "already acknowledged")
: ApplyAddComment(current, user, comment, nowUtc);
}
var audit = AppendComment(current.Comments, nowUtc, user, "Acknowledge", comment);
var next = current with
@@ -90,4 +90,16 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
statusTracker: _statusTracker,
optionsGeneration: generation);
}
/// <inheritdoc />
/// <remarks>
/// #609: without this the tracker kept a removed instance's snapshots forever (only <see cref="Create"/>
/// invalidated, and a removed instance is never re-created), and the resilience publisher re-emitted
/// them every tick — undoing the host's telemetry-hub eviction within one interval.
/// </remarks>
public void Release(string driverInstanceId)
{
_builder.Invalidate(driverInstanceId);
_statusTracker.Invalidate(driverInstanceId);
}
}
@@ -46,7 +46,7 @@ public static class LocalDbRegistration
/// Configuration key carrying the dedicated h2c sync listener's port. Zero or absent means
/// no listener is bound and the passive sync endpoint is not mapped.
/// </summary>
public const string SyncListenPortKey = "LocalDb:SyncListenPort";
public const string SyncListenPortKey = Runtime.LocalDbReplicationPolicy.SyncListenPortKey;
/// <summary>
/// The port the dedicated cleartext-HTTP/2 sync listener should bind, or <c>0</c> when the
@@ -170,9 +170,9 @@ public static class LocalDbSetup
/// <returns><c>true</c> when this node is configured to replicate.</returns>
public static bool ReplicationIsConfigured(IConfiguration configuration)
{
// Shared with the alarm-historian drain gate (#603): one predicate, so the nodes that capture
// changes and the nodes that stand down for a peer can never disagree.
return Runtime.LocalDbReplicationPolicy.IsConfigured(configuration);
// Defined beside the alarm-historian drain gate's stricter predicate (#603, review R1 m3): every node
// the gate treats as sharing rows also captures, so the two never disagree in the harmful direction.
return Runtime.LocalDbReplicationPolicy.CapturesChanges(configuration);
}
/// <summary>Opts every replicated table into change capture, baselining what is already there.</summary>
+4 -1
View File
@@ -749,7 +749,10 @@ if (LogPipelineDiagnostics.CheckFileSinkGrowth(app.Configuration,
await app.RunAsync();
}
catch (Exception ex)
// HostAbortedException is how design-time tooling (EF `dotnet ef … --startup-project`, OpenAPI generators —
// HostFactoryResolver) stops Program once it has the built host. It is not a failure: letting it reach the
// Fatal + Environment.Exit(1) below would kill the tool's own process (#618, review R1 m5).
catch (Exception ex) when (ex is not HostAbortedException)
{
// Log.Logger is still the silent bootstrap logger when the failure happened before AddZbSerilog
// ran (e.g. EffectiveRoles.Resolve), so ALSO write to stderr — the one channel that always exists.
@@ -874,8 +874,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
// (shelved, oneShot) flag pair. Timed shelving's expiry is owned by the engine; the time
// passed here is the engine's remaining time, so the node's UnshelveTime counts down to the
// real expiry (Gitea #597 — a 0 here reads as MaxTimeShelved, ~24.8 days). The SDK timer it
// arms only routes an Unshelve back to the engine (OnTimedUnshelve), a no-op once the
// engine's own expiry has run.
// arms does nothing (OnTimedUnshelve is a no-op): the engine's own expiry re-projects.
condition.SetShelvingState(
SystemContext,
shelved: state.Shelving != AlarmShelvingKind.Unshelved,
@@ -1375,25 +1374,17 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
: ("TimedShelve", DateTime.UtcNow + TimeSpan.FromMilliseconds(shelvingTime));
return HandleAlarmCommand(context, condition, operation, comment: null, unshelveAt);
};
// The auto-unshelve timer callback is SDK-initiated (the TimedShelve duration expired); the SDK
// fires it with the node manager's system context — there is NO session and NO user identity.
// Routing through HandleAlarmCommand would hit the AlarmAck gate and return BadUserAccessDenied,
// leaving the alarm permanently shelved. Instead, bypass the client gate, extract the AlarmId the
// same way HandleAlarmCommand does, and route an Unshelve command so the engine clears its shelve
// state. The manual-client Unshelve path goes through OnShelve(shelving:false) and stays gated.
alarm.OnTimedUnshelve = (context, condition) =>
{
var alarmId = condition.NodeId.Identifier?.ToString() ?? string.Empty;
// User MUST be non-empty: the engine's Part9StateMachine.ApplyUnshelve rejects a
// null/whitespace user (ArgumentException "User required."), and that exception is swallowed
// downstream — an empty user would make the timed auto-unshelve silently no-op, so a
// TimedShelve would never auto-expire. There is no client principal on a system-timer
// unshelve, so label it the canonical engine-internal "system" user (matching the engine's
// own AutoUnshelve audit user and the codebase-wide "system" system-actor convention). The
// AlarmAck gate bypass is intentional and preserved — this is a session-less SDK timer.
AlarmCommandRouter?.Invoke(new AlarmCommand(alarmId, "Unshelve", "system", null, null));
return ServiceResult.Good;
};
// The SDK's auto-unshelve timer (armed by SetShelvingState for the real remaining time since Gitea
// #597) fires with a session-less system context. It is deliberately a NO-OP: the engine owns
// timed-shelve expiry (ScriptedAlarmEngine's shelving check → Part9StateMachine.ApplyShelvingCheck,
// recorded as "AutoUnshelve" by "system" with its "Timed shelving expired" comment), and its
// re-projection is what unshelves this node. Routing an operator-style "Unshelve" from here used to
// pre-empt that check on every expiry — from BOTH pair nodes, rewriting the audit trail as a manual
// unshelve — and a timer callback already queued when a re-shelve disposed it could unshelve the
// fresh shelve. The SDK's own OnTimerExpired changes no shelving state itself (it only refreshes
// UnshelveTime after this callback), so returning without routing leaves the node shelved until
// the engine's expiry lands. Native conditions are never timed-shelved (BadNotSupported above).
alarm.OnTimedUnshelve = (_, _) => ServiceResult.Good;
// H4 — inbound Part 9 Enable/Disable (the condition type's built-in Enable/Disable methods route
// here via this delegate). The engine handles Enable/Disable for SCRIPTED alarms
@@ -1596,9 +1587,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
/// <b>Only a real change is reported.</b> The condition's gate-relevant state is read before and after
/// the SDK handler, and nothing is reported when it did not move. The SDK accepts acknowledging an
/// acknowledged condition and sets <c>Confirmed = false</c> again; a re-ack therefore also restores the
/// pre-call Confirmed and Retain, so a scripted node never drifts from its engine (which ignores a
/// re-ack) and the call is silent — unless it carried a comment, which the SDK writes and which is then
/// reported once like AddComment (Gitea #597). AddComment always reports: adding a comment is itself
/// pre-call Confirmed and Retain, so a scripted node never drifts from its engine (which ignores an
/// uncommented re-ack) and the call is silent — unless it carried a comment, which the SDK writes and
/// which is then reported once like AddComment (Gitea #597). The scripted engine records that comment
/// too (<c>Part9StateMachine.ApplyAcknowledge</c> treats a commented re-ack as AddComment), so the
/// reported comment is persisted and replicated, not node-local. AddComment always reports: adding a comment is itself
/// the change. A NATIVE re-ack is still routed to the driver: the route is fire-and-forget, so a node
/// showing Acked does not prove the upstream system acknowledged, and re-acking is the documented retry.
/// </para>
@@ -1635,7 +1628,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
// The requested duration, so UnshelveTime counts down to the real expiry (Gitea #597 — 0 reads as
// MaxTimeShelved, ~24.8 days). The scripted engine still owns the expiry and re-projects Unshelved
// when it passes (with its own remaining time meanwhile — WriteAlarmCondition); the SDK timer this
// arms only routes an Unshelve back to it (OnTimedUnshelve), a no-op once the engine has expired it.
// arms does nothing (OnTimedUnshelve is a no-op — the engine's AutoUnshelve owns the expiry).
timedShelve.OnCall = (context, method, objectId, shelvingTime) => ReportAfterMethod(
() => timed(context, method, objectId, shelvingTime),
() => alarm.SetShelvingState(SystemContext, shelved: true, oneShot: false, shelvingTime: shelvingTime));
@@ -788,9 +788,91 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// Same rationale for the ScriptedAlarm host — the bootstrap-restore path Tells it
// ApplyScriptedAlarms through the same PushDesiredSubscriptions pass.
SpawnScriptedAlarmHost();
StartPrimaryVerdictPublishing();
Bootstrap();
}
/// <summary>
/// Review R1 m2 (Gitea #597): the published Primary data-plane verdict (<see cref="IDriverPrimaryRoleSource"/>
/// — the native-ack gate and the Sparkplug primary-host claim) must mirror <see cref="ShouldServiceAsPrimary"/>
/// from the moment this host starts, not only from the first redundancy snapshot. Before this, the view sat
/// on its single-node seed (allow) through the whole boot window, so on a pair BOTH nodes accepted a native
/// ack while this host — which evaluates the member count live — dropped the upstream route on the
/// non-Primary: the acked → unacked revert #597 targets. So publish once now, and again whenever cluster
/// membership changes (a driver peer coming Up turns an unknown role from allow into deny before any
/// snapshot names this node). A non-cluster ActorRefProvider (legacy/test harnesses) just keeps the
/// initial publish.
/// </summary>
private void StartPrimaryVerdictPublishing()
{
if (_primaryRoleSource is null) return;
PublishPrimaryVerdict();
try
{
Akka.Cluster.Cluster.Get(Context.System).Subscribe(
Self, Akka.Cluster.ClusterEvent.SubscriptionInitialStateMode.InitialStateAsSnapshot,
typeof(Akka.Cluster.ClusterEvent.IMemberEvent));
_subscribedToMemberEvents = true;
}
catch (Exception)
{
// Non-cluster ActorRefProvider — no membership to follow; DriverMemberCount() reads 0 there too.
}
}
/// <summary>A cluster member changed status: re-publish the Primary verdict now and once more shortly after.
/// The deferred pass covers the cluster read-view (what <see cref="DriverMemberCount"/> reads) updating on its
/// own subscriber AFTER this event was delivered here — the two are not ordered.</summary>
private void OnDriverMembershipChanged(Akka.Cluster.ClusterEvent.IMemberEvent _)
{
PublishPrimaryVerdictIfChanged();
// One keyed timer: a burst of member events collapses into a single deferred pass.
Timers.StartSingleTimer("republish-primary-verdict", RepublishPrimaryVerdict.Instance, MembershipRepublishDelay);
}
/// <summary>Publishes <see cref="ShouldServiceAsPrimary"/> to the primary-role source (no-op when none is wired).
/// The snapshot path publishes on every snapshot, as the other verdicts do.</summary>
private void PublishPrimaryVerdict()
{
if (_primaryRoleSource is null) return;
var verdict = ShouldServiceAsPrimary();
_lastPublishedPrimaryVerdict = verdict;
_primaryRoleSource.PublishPrimaryRole(verdict);
}
/// <summary>The membership path publishes only a CHANGED verdict — member events are frequent and carry no
/// role information, so republishing an unchanged verdict on each would be noise.</summary>
private void PublishPrimaryVerdictIfChanged()
{
if (_primaryRoleSource is null || ShouldServiceAsPrimary() == _lastPublishedPrimaryVerdict) return;
PublishPrimaryVerdict();
}
/// <inheritdoc />
protected override void PostStop()
{
if (_subscribedToMemberEvents)
{
try { Akka.Cluster.Cluster.Get(Context.System).Unsubscribe(Self); }
catch (Exception) { /* system shutting down */ }
}
base.PostStop();
}
/// <summary>Self-message: re-evaluate and re-publish the Primary verdict (see <see cref="OnDriverMembershipChanged"/>).</summary>
internal sealed class RepublishPrimaryVerdict
{
/// <summary>The shared instance.</summary>
public static readonly RepublishPrimaryVerdict Instance = new();
private RepublishPrimaryVerdict() { }
}
/// <summary>How long after a membership event the Primary verdict is re-published a second time.</summary>
private static readonly TimeSpan MembershipRepublishDelay = TimeSpan.FromSeconds(1);
private bool _subscribedToMemberEvents;
private bool? _lastPublishedPrimaryVerdict;
/// <summary>
/// Spawns the single <see cref="VirtualTagHostActor"/> child that owns the Equipment-namespace
/// VirtualTagActors and bridges their results onto the OPC UA publish actor. A test-supplied
@@ -990,12 +1072,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
var revision = RevisionHash.Parse(cached.RevisionHash);
_currentRevision = revision;
BecomeSteady();
// Restore drivers + address space + subscriptions from the cached bytes — no SQL read, no
// re-ack (the deployment is already Applied). Same in-hand-bytes apply as the Direct-mode
// cache boot, but this is the steady-state config source, not an outage fallback.
ApplyCachedArtifact(deploymentId, cached.Artifact);
if (!ApplyCachedArtifact(deploymentId, cached.Artifact))
{
// #601 / review R1 m4: unusable cached bytes applied nothing, so they must not become the
// current revision — that would ack a re-dispatch of the same revision as already applied
// without ever applying it. Steady-with-no-revision, exactly as for an empty cache.
_log.Warning(
"DriverHost {Node}: FetchAndCache boot — the cached artifact for deployment {Id} (rev {Rev}) " +
"is unreadable; entering Steady (no revision). The next dispatch will fetch.",
_localNode, deploymentId, revision);
BecomeSteady();
return;
}
_currentRevision = revision;
BecomeSteady();
_log.Info(
"DriverHost {Node}: FetchAndCache boot — restored served state for deployment {Id} " +
@@ -1059,14 +1153,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
var revision = RevisionHash.Parse(cached.RevisionHash);
// #601 / review R1 m4: unusable cached bytes applied nothing — the node is NOT serving a
// configuration, so it must not claim RUNNING FROM CACHE, adopt the revision (a re-dispatch of
// it would then be acked without being applied) or skip Stale's ConfigDb retry loop.
if (!ApplyCachedArtifact(deploymentId, cached.Artifact))
{
_log.Error(
"DriverHost {Node}: the cached artifact for deployment {Id} (rev {Rev}) is unreadable; " +
"not booting from it — entering Stale with no configuration.",
_localNode, deploymentId, revision);
return false;
}
// Steady, not Stale: this node is serving a real configuration. The retry-db timer that
// Stale would have started does not run here, so recovery rides on the next dispatch —
// matching how a normally-booted node behaves.
_currentRevision = revision;
BecomeSteady();
ApplyCachedArtifact(deploymentId, cached.Artifact);
_isRunningFromCache = true;
_log.Warning(
"DriverHost {Node}: RUNNING FROM CACHE — central ConfigDb is unreachable, so this node " +
@@ -1094,16 +1198,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <c>UpsertNodeDeploymentState</c> and <c>SendAck</c> for the same reason: both write to or
/// depend on central.
/// </remarks>
private void ApplyCachedArtifact(DeploymentId deploymentId, byte[] blob)
/// <returns><see langword="false"/> when the bytes were empty or malformed and nothing was applied — the
/// caller must not adopt the cached revision (#601, review R1 m4).</returns>
private bool ApplyCachedArtifact(DeploymentId deploymentId, byte[] blob)
{
var correlation = CorrelationId.NewId();
// #601: null = the cached bytes were empty or malformed — no answer, not an empty configuration.
// Keep last-known-good: no address-space rebuild and no subscription push from unusable bytes.
// #601: null = the cached bytes were empty or malformed — no answer, not an empty configuration.
// Keep last-known-good: no address-space rebuild and no subscription push from unusable bytes.
if (ReconcileDriversFromBlob(deploymentId, blob) is null)
return;
return false;
// Hand the cached blob to the rebuild so it materialises the client-facing address space from
// it directly. Passing only the deploymentId would drive a ConfigDb read — unreachable here by
@@ -1113,6 +1217,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, blob));
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
return true;
}
/// <summary>
@@ -1170,6 +1275,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
@@ -1228,6 +1336,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
@@ -1975,7 +2086,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// drain verdict published immediately above. A driver reading this claims an exclusive identity
// on a shared external system (Gitea #511: the Sparkplug host id), where two claimants is the
// harm, so an unknown role with a driver peer present must deny.
_primaryRoleSource?.PublishPrimaryRole(ShouldServiceAsPrimary());
PublishPrimaryVerdict();
}
private void Stale()
@@ -1998,6 +2109,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
@@ -3233,7 +3347,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Context.Stop(entry.Actor);
_children.Remove(driverInstanceId);
// #609: the instance left the plan (StopChild is only reached via plan.ToStop), so its cached
// Health/Resilience snapshots must not replay to new telemetry subscribers.
// Health/Resilience snapshots must not replay to new telemetry subscribers. Release the shared
// resilience state FIRST: the resilience publisher re-emits the tracker's snapshots every tick, so
// evicting the hub alone was undone within one interval.
_invokerFactory.Release(driverInstanceId);
_telemetryHub?.Evict(driverInstanceId);
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}
@@ -3,26 +3,36 @@ using Microsoft.Extensions.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Runtime;
/// <summary>
/// The ONE definition of "this node's LocalDb is replicated to a peer" (#603). Two consumers must agree
/// on it, and used to carry separate predicates that diverged: the Host's table-registration step
/// (<c>LocalDbSetup.ReplicationIsConfigured</c> - which nodes capture changes) and the alarm-historian
/// drain gate (<c>AddAlarmHistorian</c> - which nodes stand down in favour of a peer). Lives in Runtime
/// because Host references Runtime and not the reverse.
/// The definitions of "this node's LocalDb is replicated to a peer" (#603) — one place, two deliberately
/// different predicates for the two consumers: the Host's table-registration step
/// (<c>LocalDbSetup.ReplicationIsConfigured</c> → <see cref="CapturesChanges"/>: which nodes capture
/// changes) and the alarm-historian drain gate (<c>AddAlarmHistorian</c> → <see cref="SharesRowsWithPeer"/>:
/// which nodes stand down in favour of a peer). Lives in Runtime because Host references Runtime and not
/// the reverse.
/// </summary>
/// <remarks>
/// <para>
/// <b>Predicate: <c>LocalDb:Replication:PeerAddress</c> OR <c>LocalDb:Replication:ApiKey</c> is
/// non-blank.</b> The OR is required: only the dialling half of a pair sets <c>PeerAddress</c>, while
/// the passive half carries the shared <c>ApiKey</c> (plus a listen port) and nothing else.
/// <b>Why two predicates (review R1 m3).</b> The two answers cost differently when wrong. Registering a
/// node that has no working sync path only costs capture triggers. Gating the drain of a node that
/// shares its queue with nobody parks its alarm events behind a Primary that does not hold them —
/// undelivered until the node is promoted, and drop-oldest at <c>Capacity</c> meanwhile. So
/// registration stays permissive and the gate is strict. Every node the gate treats as sharing also
/// captures, so the two never disagree in the harmful direction.
/// </para>
/// <para>
/// <b>Why the drain gate takes the same predicate (it previously used PeerAddress OR
/// <c>SyncListenPort</c>).</b> The gate's question is "does a peer hold the same rows?". The passive
/// sync endpoint is fail-closed on the key, so a node with a listen port but no <c>ApiKey</c> rejects
/// every inbound stream and shares its queue with nobody - gating it would park its alarm events
/// behind a Primary that cannot deliver them. Conversely an <c>ApiKey</c>-only passive node IS
/// registered and does receive its peer's rows, so it must be gated or both halves drain. The port
/// is not evidence of a peer; the key is.
/// <b>Registration: <c>LocalDb:Replication:PeerAddress</c> OR <c>LocalDb:Replication:ApiKey</c>.</b>
/// The OR is required: only the dialling half of a pair sets <c>PeerAddress</c>, while the passive
/// half carries the shared <c>ApiKey</c> (plus a listen port) and nothing else. Keying on
/// <c>PeerAddress</c> alone would strip capture from every passive node.
/// </para>
/// <para>
/// <b>Drain gate: <c>ApiKey</c> AND (<c>PeerAddress</c> OR <c>LocalDb:SyncListenPort</c> &gt; 0).</b> A
/// peer holds this node's rows only over a working sync path: a dial (<c>PeerAddress</c>) or a
/// listener (<c>SyncListenPort</c>), each authenticated by the key. Both auth directions fail closed,
/// so a path without the key shares nothing, and a key with neither path (e.g. a fleet-wide
/// secret/env <c>ApiKey</c> on a node with no peer) shares nothing either. Every docker-dev shape
/// gives the same answer under both predicates: site-a-1 (PeerAddress + ApiKey + port) and site-a-2
/// (ApiKey + port) are gated; central and site-b (neither) are not.
/// </para>
/// </remarks>
public static class LocalDbReplicationPolicy
@@ -30,14 +40,31 @@ public static class LocalDbReplicationPolicy
/// <summary>Configuration section holding the LocalDb replication options.</summary>
public const string ReplicationSectionPath = "LocalDb:Replication";
/// <summary>Whether this node's LocalDb replicates to a pair peer.</summary>
/// <summary>The dedicated passive-sync listener port key (same key as <c>LocalDbRegistration.SyncListenPortKey</c>).</summary>
public const string SyncListenPortKey = "LocalDb:SyncListenPort";
/// <summary>Whether this node registers its replicated tables for change capture (table registration).</summary>
/// <param name="configuration">The application configuration.</param>
/// <returns><c>true</c> when <c>PeerAddress</c> or <c>ApiKey</c> is set under the replication section.</returns>
public static bool IsConfigured(IConfiguration configuration)
public static bool CapturesChanges(IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(configuration);
var section = configuration.GetSection(ReplicationSectionPath);
return !string.IsNullOrWhiteSpace(section["PeerAddress"])
|| !string.IsNullOrWhiteSpace(section["ApiKey"]);
}
/// <summary>Whether a pair peer actually holds this node's replicated rows (the alarm drain gate).</summary>
/// <param name="configuration">The application configuration.</param>
/// <returns><c>true</c> when <c>ApiKey</c> is set AND either <c>PeerAddress</c> is set or
/// <c>LocalDb:SyncListenPort</c> is a positive port.</returns>
public static bool SharesRowsWithPeer(IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(configuration);
var section = configuration.GetSection(ReplicationSectionPath);
if (string.IsNullOrWhiteSpace(section["ApiKey"])) return false;
var dials = !string.IsNullOrWhiteSpace(section["PeerAddress"]);
var listens = int.TryParse(configuration[SyncListenPortKey], out var port) && port > 0;
return dials || listens;
}
}
@@ -177,11 +177,13 @@ public static class ServiceCollectionExtensions
// possibly deliver its events. Scoping the gate to "is my queue actually shared?" is what
// keeps the two scopes from disagreeing.
// BOTH halves of a pair share the queue, but only one of them dials: the initiator sets
// Replication:PeerAddress, its partner carries the shared ApiKey and waits. The predicate is
// LocalDbReplicationPolicy - the same one LocalDbSetup uses to decide which nodes capture
// changes at all. It deliberately does NOT count SyncListenPort: the passive endpoint is
// fail-closed on the key, so a port with no ApiKey shares nothing and must keep draining.
var replicated = LocalDbReplicationPolicy.IsConfigured(configuration);
// Replication:PeerAddress, its partner carries the shared ApiKey plus SyncListenPort and waits.
// The predicate is LocalDbReplicationPolicy.SharesRowsWithPeer: ApiKey AND (PeerAddress OR
// SyncListenPort > 0) - an authenticated sync path in either direction. Both auth directions
// fail closed, so a path without the key, or a key with no path (a fleet-wide secret on an
// unpaired node), shares nothing and must keep draining. Deliberately stricter than the
// registration predicate (CapturesChanges) - see the policy's remarks.
var replicated = LocalDbReplicationPolicy.SharesRowsWithPeer(configuration);
if (!replicated)
{