@@ -29,7 +29,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
/// back on its own recovers in place.
/// </para>
/// </remarks>
public sealed class OpcUaClientDriver : IDriver , ITagDiscovery , IReadable , IWritable , ISubscribable , IHostConnectivityProbe , IAlarmSource , ISkippedTagReporter , IDisposable , IAsyncDisposable
public sealed class OpcUaClientDriver : IDriver , ITagDiscovery , IReadable , IWritable , ISubscribable , IHostConnectivityProbe , IAlarmSource , ISkippedTagReporter , IDriverConnectionLossSource , IDisposable , IAsyncDisposable
{
/// <summary>Authored tags whose TagConfig carried no usable <c>nodeId</c>, so there is no upstream node
/// to dial (Gitea #541). This driver's skip path was the quietest of the twelve: <c>NamespaceMap</c>
@@ -83,6 +83,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
_logger = logger ? ? NullLogger < OpcUaClientDriver > . Instance ;
_secretResolver = secretResolver ? ? NullSecretResolver . Instance ;
_sdkTelemetry = logger is null ? SdkTelemetry . Context : SdkTelemetry . ForLogger ( logger ) ;
PostRecoveryReconcileDelay = ( ) = > _options . Timeout + _options . KeepAliveInterval ;
}
/// <summary>Mutable so <see cref="InitializeAsync"/> can adopt a re-parsed config on reinitialize
@@ -135,6 +136,14 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>Occurs when the host connectivity status of the remote OPC UA server changes.</summary>
public event EventHandler < HostStatusChangedEventArgs > ? OnHostStatusChanged ;
/// <inheritdoc />
/// <remarks>Gitea #676 — raised at most once per session, and only on a DEFINITIVE loss
/// (<see cref="IsDefinitiveSessionLoss"/>, or a tracked subscription the SERVER reported dropped — never the
/// publishing-stall heuristic). Re-armed when the session is released (re-initialise / shutdown). Known gap: the host
/// acts on it only while Connected, so a raise it receives before it has processed the successful initialise is
/// swallowed and that session falls back to the #527 watchdog.</remarks>
public event EventHandler < DriverConnectionLossEventArgs > ? ConnectionLost ;
// OPC UA StatusCode constants the driver surfaces for local-side faults. Upstream-server
// StatusCodes are passed through verbatim per driver-specs.md §8 "cascading quality" —
// downstream clients need to distinguish 'remote source down' from 'local driver failure'.
@@ -164,6 +173,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>Test-only seam (#670): the most recent in-place recovery task.</summary>
internal Task RecoveryTaskForTest = > _recoveryTask ;
/// <summary>Test seam (#676): the trailing reconcile read scheduled by the last in-place recovery.</summary>
internal Task ReconcileTaskForTest = > _reconcileTask ;
/// <summary>Test-only seam (#670 review M1): raise the session's PublishError as the SDK publish loop does.</summary>
/// <param name="status">The publish error status.</param>
/// <param name="subscriptionId">The subscription id the error names (0 = none).</param>
@@ -282,6 +294,25 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>The last in-place recovery started (#670). Test seam: lets a test await it deterministically.</summary>
private Task _recoveryTask = Task . CompletedTask ;
/// <summary>#676 — 1 once <see cref="ConnectionLost"/> has been raised for the CURRENT session (once per session);
/// cleared when the session is released, so a re-initialised driver reports its next loss.</summary>
private int _connectionLostRaised ;
/// <summary>#676 — the trailing reconcile read scheduled by the last in-place recovery. Test seam.</summary>
private Task _reconcileTask = Task . CompletedTask ;
/// <summary>#676 — late notifications the replay guard dropped since it was armed, and the refs they were for;
/// summarised in ONE Information line when the window disarms. Guarded by <see cref="_replayGuardLock"/>.</summary>
private int _replayDrops ;
private readonly HashSet < string > _replayDropRefs = new ( StringComparer . Ordinal ) ;
/// <summary>
/// Test seam (#676): how long after an in-place recovery the trailing reconcile read runs. Production: the
/// operation timeout plus one keep-alive interval — by then every request that was in flight across the stall
/// (including the Publish responses whose notifications the SDK can lose) has completed or timed out.
/// </summary>
internal Func < TimeSpan > PostRecoveryReconcileDelay { get ; set ; }
/// <summary>Test seam (#670 review M1): whether a subscription has stopped publishing. Production asks the SDK.</summary>
internal Func < Subscription , bool > IsPublishingStopped { get ; set ; } = static s = > s . Created & & s . PublishingStopped ;
@@ -326,6 +357,37 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>Serialises arming against expiry, so an expiry racing a re-arm cannot wipe the new seeds.</summary>
private readonly object _replayGuardLock = new ( ) ;
/// <summary>
/// Review of 3b88056f — serialises "decide whether this upstream value may be published" with "publish it" across
/// the three paths that publish upstream data: live notifications, the recovery read and the trailing reconcile
/// read. Without it a path could pass its newer-than check, lose the CPU to another path publishing a NEWER value,
/// and then deliver its older one last — regressing the node. It also covers the comms-loss Bad fan
/// (<see cref="PublishCommsLoss"/>). One driver-wide lock rather than one per ref: it also serialises delivery
/// ACROSS subscriptions — with several subscriptions the SDK dispatches each subscription's messages on its own
/// worker, so their notifications no longer run in parallel. That is the cost, and it is small: each critical
/// section is a dictionary update plus a subscriber call that only <c>Tell</c>s an actor (tens of nanoseconds to
/// a few microseconds), nothing awaits under it, and per-subscription message order is unaffected. Because
/// subscribers are invoked under it, a subscriber must never block on, or call back into, this driver.
/// </summary>
private readonly Lock _deliveryLock = new ( ) ;
/// <summary>
/// Re-review of 30a9d5f0 — per RawPath, a counter bumped (under <see cref="_deliveryLock"/>) every time a value is
/// delivered for it. The trailing re-read captures it before issuing its read and skips any ref delivered to since:
/// without source timestamps the read's value cannot be ordered against a notification that landed while the read
/// was in flight, and publishing it would regress the node to the value the read sampled.
/// </summary>
private readonly System . Collections . Concurrent . ConcurrentDictionary < string , long > _deliverySeq =
new ( StringComparer . Ordinal ) ;
/// <summary>Review of 3b88056f — cancels the post-recovery background work (trailing read, replay-guard disarm timer)
/// of the current session; cancelled and replaced whenever the session is released, and on dispose.</summary>
private CancellationTokenSource _sessionWorkCts = new ( ) ;
/// <summary>Test seam (review of 3b88056f): invoked inside the trailing read's critical section, after it decided
/// to publish a ref and before it does — where a racing notification would otherwise overtake it.</summary>
internal Action < string > ? BeforeReconcilePublishForTest { get ; set ; }
/// <summary>Floor of the replay-guard window; the live gate saw the late replay 4–
private static readonly TimeSpan ReplayGuardMinimum = TimeSpan . FromSeconds ( 30 ) ;
@@ -883,9 +945,40 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
$"No endpoint at '{endpointUrl}' matches SecurityPolicy={policy} + SecurityMode={mode}. " +
$"Server advertises: {advertised}" ) ;
}
// #676: connect where the operator pointed us, not where the server says it lives.
match . EndpointUrl = RewriteEndpointUrl ( match . EndpointUrl , endpointUrl ) ;
return match ;
}
/// <summary>
/// Gitea #676 — the address to open the session on. A server reports its endpoints with ITS view of its own
/// address; behind NAT, a port map, a container port publish or a proxy that address is unreachable from here
/// (or bypasses the proxy). When the advertised URL's scheme matches the configured one, the configured host
/// AND port (when it names one) replace the advertised ones and the advertised path is kept — the SDK's own
/// <c>CoreClientUtils.SelectEndpoint</c> rule, which this driver's policy-aware selection bypassed. A different
/// scheme is a different transport and is left alone; an unparseable advertised URL falls back to the
/// configured one.
/// </summary>
/// <param name="advertisedUrl">The <c>EndpointUrl</c> the server returned from GetEndpoints.</param>
/// <param name="configuredUrl">The URL the driver was configured with (the one GetEndpoints was sent to).</param>
/// <returns>The URL to connect the session to.</returns>
internal static string RewriteEndpointUrl ( string? advertisedUrl , string configuredUrl )
{
if ( ! Uri . TryCreate ( configuredUrl , UriKind . Absolute , out var configured ) ) return advertisedUrl ? ? configuredUrl ;
if ( string . IsNullOrWhiteSpace ( advertisedUrl )
| | ! Uri . TryCreate ( advertisedUrl , UriKind . Absolute , out var advertised ) )
return configuredUrl ;
if ( ! string . Equals ( advertised . Scheme , configured . Scheme , StringComparison . OrdinalIgnoreCase ) ) return advertisedUrl ;
// A port-less configured URL (Uri.Port == -1 for opc.tcp) keeps the advertised port — copying -1 would drop it
// and the SDK would fall back to 4840 (review of 3b88056f; same fix as the client's EndpointSelector).
var builder = new UriBuilder ( advertised )
{
Host = configured . DnsSafeHost ,
Port = configured . Port > 0 ? configured . Port : advertised . Port ,
} ;
return builder . Uri . ToString ( ) ;
}
/// <summary>
/// Build a <see cref="UserIdentity"/> carrying a client user-authentication
/// certificate loaded from <see cref="OpcUaClientDriverOptions.UserCertificatePath"/>.
@@ -986,6 +1079,20 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
Interlocked . Exchange ( ref _sessionLost , 0 ) ;
Interlocked . Exchange ( ref _subscriptionLost , 0 ) ;
Interlocked . Exchange ( ref _publishingStoppedSinceTicks , 0 ) ;
// #676: re-arm the definitive-loss signal for the next session.
Interlocked . Exchange ( ref _connectionLostRaised , 0 ) ;
// Review of 3b88056f: the released session's post-recovery timers must not outlive it.
CancelSessionWork ( ) ;
}
/// <summary>Cancel the current session's background work and start a fresh token for the next one.</summary>
private void CancelSessionWork ( )
{
// Cancelled but deliberately NOT disposed (re-review of 30a9d5f0): the keep-alive thread may be reading the old
// source's Token concurrently, and a disposed source throws ObjectDisposedException there. Nothing here is
// timer-backed (no CancelAfter), so an undisposed source holds no unmanaged resource.
var old = Interlocked . Exchange ( ref _sessionWorkCts , new CancellationTokenSource ( ) ) ;
old . Cancel ( ) ;
}
/// <summary>Upper bound on the best-effort wire calls a teardown makes against a possibly-dead upstream.</summary>
@@ -1021,8 +1128,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
// previous connection's value bookkeeping applies to them.
_lastValues . Clear ( ) ;
_lastGoodSourceTicks . Clear ( ) ;
_replayGuard . Clear ( ) ;
Interlocked . Exchange ( ref _replayGuardUntilTicks , 0 ) ;
lock ( _replayGuardLock )
{
_replayGuard . Clear ( ) ;
Interlocked . Exchange ( ref _replayGuardUntilTicks , 0 ) ;
_replayDrops = 0 ;
_replayDropRefs . Clear ( ) ;
}
_awaitingRecovery . Clear ( ) ;
// #619 re-review: the last-told alarm states (and ack targets) deliberately SURVIVE a shutdown. A
// ReinitializeAsync (comms-loss watchdog / Force Reconnect) is followed by the host re-subscribing the
@@ -1762,20 +1874,33 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
StatusCode : dv . StatusCode . Code ,
SourceTimestampUtc : dv . SourceTimestamp = = DateTime . MinValue ? null : dv . SourceTimestamp ,
ServerTimestampUtc : dv . ServerTimestamp = = DateTime . MinValue ? DateTime . UtcNow : dv . ServerTimestamp ) ;
// F2: inside the post-recovery replay window only, a late OLDER notification must not regress the node.
if ( IsLateReplay ( fullRef , snapshot . SourceTimestampUtc , out var newest ) )
// Review of 3b88056f: the replay check and the publish are one critical section shared with the recovery and
// trailing reads, so an older value can never be delivered after a newer one (see _deliveryLock).
lock ( _deliveryLock )
{
_diag . Record ( OpcUaClientDiagnostics . StaleNotificationDropped ) ;
_logger . LogDebug (
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
"(source {Source:O} is not newer than {Newest:O})." , _driverInstanceId , fullRef , snapshot . SourceTimestampUtc ,
new DateTime ( newest , DateTimeKind . Utc ) ) ;
return ;
// F2: inside the post-recovery replay window only, a late OLDER notification must not regress the node.
if ( IsLateReplay ( fullRef , snapshot . SourceTimestampUtc , out var newest ) )
{
_diag . Record ( OpcUaClientDiagnostics . StaleNotificationDropped ) ;
// #676: the OTel counter and the Debug line below are invisible on a default-configured node; the
// per-recovery Information summary logged at disarm is what makes the guard observable.
lock ( _replayGuardLock )
{
_replayDrops + + ;
_replayDropRefs . Add ( fullRef ) ;
}
_logger . LogDebug (
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
"(source {Source:O} is not newer than {Newest:O})." , _driverInstanceId , fullRef , snapshot . SourceTimestampUtc ,
new DateTime ( newest , DateTimeKind . Utc ) ) ;
return ;
}
RecordUpstreamGood ( fullRef , snapshot ) ;
_lastValues [ fullRef ] = snapshot ;
_deliverySeq . AddOrUpdate ( fullRef , 1 , ( _ , n ) = > n + 1 ) ;
_awaitingRecovery . TryRemove ( fullRef , out _ ) ;
OnDataChange ? . Invoke ( this , new DataChangeEventArgs ( handle , fullRef , snapshot ) ) ;
}
RecordUpstreamGood ( fullRef , snapshot ) ;
_lastValues [ fullRef ] = snapshot ;
_awaitingRecovery . TryRemove ( fullRef , out _ ) ;
OnDataChange ? . Invoke ( this , new DataChangeEventArgs ( handle , fullRef , snapshot ) ) ;
}
/// <summary>#670 F2 — remember the last accepted Good upstream source time for <paramref name="fullRef"/>.</summary>
@@ -1798,14 +1923,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
if ( until = = 0 ) return false ;
if ( Clock . GetUtcNow ( ) . UtcDateTime . Ticks > = until )
{
lock ( _replayGuardLock )
{
if ( Interlocked . Read ( ref _replayGuardUntilTicks ) = = until )
{
Interlocked . Exchange ( ref _replayGuardUntilTicks , 0 ) ;
_replayGuard . Clear ( ) ;
}
}
DisarmReplayGuard ( until ) ;
return false ;
}
if ( source is not { } src ) return false ;
@@ -1821,11 +1939,40 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
}
/// <summary>
/// Expire the replay guard armed until <paramref name="until"/> — a no-op when it was re-armed or already disarmed
/// since — and log ONE Information summary of what it dropped (#676). The guard's OTel counter and per-drop Debug
/// line are invisible on a default-configured node, so before this a recovery's drops left no trace at all.
/// </summary>
/// <param name="until">The expiry the caller observed; only that arming is disarmed.</param>
private void DisarmReplayGuard ( long until )
{
int drops , refs ;
lock ( _replayGuardLock )
{
if ( Interlocked . Read ( ref _replayGuardUntilTicks ) ! = until ) return ;
Interlocked . Exchange ( ref _replayGuardUntilTicks , 0 ) ;
_replayGuard . Clear ( ) ;
drops = _replayDrops ;
refs = _replayDropRefs . Count ;
_replayDrops = 0 ;
_replayDropRefs . Clear ( ) ;
}
LogReplaySummary ( drops , refs ) ;
}
/// <summary>One line per replay window: Information when the guard dropped something, Debug otherwise.</summary>
private void LogReplaySummary ( int drops , int refs ) = >
_logger . Log ( drops > 0 ? LogLevel . Information : LogLevel . Debug ,
"OpcUaClientDriver '{DriverInstanceId}': post-recovery replay window closed — dropped {Drops} late " +
"notifications across {Refs} refs after recovery." , _driverInstanceId , drops , refs ) ;
/// <summary>
/// #670 F2 — arm the replay guard on a same-session recovery: every ref with a known Good upstream source time,
/// for max(30 s, 2 ×
/// </summary>
private void ArmReplayGuard ( DateTime now )
/// <returns>The window's expiry in clock ticks.</returns>
private long ArmReplayGuard ( DateTime now )
{
var window = ReplayGuardMinimum ;
foreach ( var sub in _subscriptions . Values . Select ( r = > r . Subscription ) )
@@ -1833,12 +1980,39 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
var keepAliveWindow = TimeSpan . FromMilliseconds ( sub . CurrentPublishingInterval * ( double ) sub . CurrentKeepAliveCount ) ;
if ( keepAliveWindow * 2 > window ) window = keepAliveWindow * 2 ;
}
int supersededDrops , supersededRefs ;
lock ( _replayGuardLock )
{
// A re-arm inside an open window ends that window: report what it dropped and count afresh.
supersededDrops = _replayDrops ;
supersededRefs = _replayDropRefs . Count ;
_replayDrops = 0 ;
_replayDropRefs . Clear ( ) ;
_replayGuard . Clear ( ) ;
foreach ( var kv in _lastGoodSourceTicks ) _replayGuard [ kv . Key ] = kv . Value ;
Interlocked . Exchange ( ref _replayGuardUntilTicks , ( now + window ) . Ticks ) ;
}
if ( supersededDrops > 0 ) LogReplaySummary ( supersededDrops , supersededRefs ) ;
return ( now + window ) . Ticks ;
}
/// <summary>
/// #676 — disarm the replay guard (and log its summary) when its window ends, even if no notification arrives to
/// disarm it lazily in <see cref="IsLateReplay"/> — a quiet subscription would otherwise never log the summary.
/// </summary>
private async Task DisarmReplayGuardWhenDueAsync ( long until , CancellationToken ct )
{
try
{
var wait = new DateTime ( until , DateTimeKind . Utc ) - Clock . GetUtcNow ( ) . UtcDateTime ;
if ( wait > TimeSpan . Zero ) await Task . Delay ( wait + TimeSpan . FromMilliseconds ( 100 ) , Clock , ct ) . ConfigureAwait ( false ) ;
if ( Clock . GetUtcNow ( ) . UtcDateTime . Ticks > = until ) DisarmReplayGuard ( until ) ;
}
catch ( OperationCanceledException ) when ( ct . IsCancellationRequested ) { /* session released */ }
catch ( Exception ex )
{
_logger . LogDebug ( ex , "OpcUaClientDriver '{DriverInstanceId}': replay-guard disarm timer faulted." , _driverInstanceId ) ;
}
}
/// <summary>
@@ -2683,7 +2857,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// session-recreate path cloned subscriptions the driver never re-tracked, and it kept health Healthy while
/// it did so — the #670 symptom. A session that does not come back keeps health Degraded with the read stamp
/// frozen, which is exactly what the host's #527 comms-loss watchdog trips on: ShutdownAsync + InitializeAsync
/// + re-subscribe, whose initial values come back Good.
/// + re-subscribe, whose initial values come back Good. When the keep-alive status proves the session is gone
/// for good (<see cref="IsDefinitiveSessionLoss"/>) the driver also raises <see cref="ConnectionLost"/> (#676),
/// so the host runs that sequence at once instead of after the watchdog's silence window.
/// </remarks>
private void OnKeepAlive ( ISession sender , KeepAliveEventArgs e )
{
@@ -2706,18 +2882,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
UpdateHealth ( _ = > new DriverHealth ( DriverState . Healthy , now , null ) ) ;
// A fresh grace: every subscription reports stopped right after an outage until its next message.
Interlocked . Exchange ( ref _publishingStoppedSinceTicks , 0 ) ;
ArmReplayGuard ( now ) ;
var guardUntil = ArmReplayGuard ( now ) ;
_diag . Record ( OpcUaClientDiagnostics . KeepAliveRecovered ) ;
_logger . LogInformation (
"OpcUaClientDriver '{DriverInstanceId}': upstream keep-alive recovered on the same session; " +
"re-reading the refs published Bad during the outage." , _driverInstanceId ) ;
_recoveryTask = Task . Run ( ( ) = > RecoverAfterOutageAsync ( sender ) ) ;
var recovery = Task . Run ( ( ) = > RecoverAfterOutageAsync ( sender ) ) ;
_recoveryTask = recovery ;
var sessionWork = _sessionWorkCts . Token ;
_reconcileTask = ReconcileAfterRecoveryAsync ( sender , recovery , PostRecoveryReconcileDelay ( ) , sessionWork ) ;
_ = DisarmReplayGuardWhenDueAsync ( guardUntil , sessionWork ) ;
return ;
}
if ( FindStalledSubscription ( now ) is { } stalled )
{
DeclareSubscriptionLost ( $"subscription '{stalled}' stopped publishing while the session stayed up" , now ) ;
DeclareSubscriptionLost ( $"subscription '{stalled}' stopped publishing while the session stayed up" , now ,
serverReported : false ) ;
return ;
}
@@ -2729,14 +2910,56 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
TransitionTo ( HostState . Stopped ) ;
_diag . Record ( OpcUaClientDiagnostics . KeepAliveMiss ) ;
UpdateHealth ( h = > new DriverHealth ( DriverState . Degraded , h . LastSuccessfulRead , $"keep-alive: {e.Status}" ) ) ;
if ( Interlocked . Exchange ( ref _sessionLost , 1 ) = = 1 ) return ;
if ( Volatile . Read ( ref _subscriptionLost ) = = 1 ) return ; // already published Bad for this connection
// #676: the first bad keep-alive is often ambiguous and a definitive one follows, so this is checked on EVERY
// bad keep-alive — but only after the outage below has been published, so the Bad reaches subscribers first.
var keepAliveCode = e . Status ? . StatusCode ? ? StatusCodes . Good ;
var definitive = IsDefinitiveSessionLoss ( keepAliveCode ) ;
// A subscription loss already published Bad for this connection.
if ( Interlocked . Exchange ( ref _sessionLost , 1 ) = = 0 & & Volatile . Read ( ref _subscriptionLost ) = = 0 )
{
_logger . LogWarning (
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver." ,
_driverInstanceId , e . Status ) ;
PublishCommsLoss ( now ) ;
}
if ( definitive )
RaiseConnectionLost ( $"upstream session keep-alive reported {StatusCodes.GetBrowseName(keepAliveCode.Code)}" ) ;
}
/// <summary>
/// #676 — keep-alive statuses that prove the session can NEVER recover in place, so waiting out the #527
/// watchdog would only delay the inevitable re-initialise. The server forgot the session
/// (<c>BadSessionIdInvalid</c> — what a re-opened channel to a restarted server answers), closed it
/// (<c>BadSessionClosed</c>) or never activated it (<c>BadSessionNotActivated</c>); or the server HALTED
/// (<c>BadServerHalted</c>) — measured on the #676 restart wire test, the SDK never re-opens a channel the
/// server halted: 15 s after the server was back the same session still failed every request with
/// <c>BadServerHalted</c>, so a halted session is dead for good. Everything else — <c>BadNoCommunication</c>,
/// <c>BadConnectionClosed</c>, <c>BadNotConnected</c>, request timeouts — is AMBIGUOUS (a stalled link that
/// resumes keeps its session, see the stall wire test) and stays with the watchdog.
/// </summary>
/// <param name="status">A keep-alive status.</param>
/// <returns>True when the session is definitively gone.</returns>
internal static bool IsDefinitiveSessionLoss ( StatusCode status ) = >
status = = StatusCodes . BadSessionIdInvalid
| | status = = StatusCodes . BadSessionClosed
| | status = = StatusCodes . BadSessionNotActivated
| | status = = StatusCodes . BadServerHalted ;
/// <summary>#676 — raise <see cref="ConnectionLost"/> once for the current session, isolating a faulting subscriber.</summary>
/// <param name="reason">Why the connection is definitively gone, for the host's log.</param>
private void RaiseConnectionLost ( string reason )
{
if ( Interlocked . Exchange ( ref _connectionLostRaised , 1 ) = = 1 ) return ;
_logger . LogWarning (
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
"a re BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver." ,
_driverInstanceId , e . Status ) ;
PublishCommsLoss ( now ) ;
"OpcUaClientDriver '{DriverInstanceId}': connection definitively lost ({Reason}); asking the host to " +
"re-initialise now instead of waiting for the comms-loss watchdog." , _driverInstanceId , reason ) ;
try { ConnectionLost ? . Invoke ( this , new DriverConnectionLossEventArgs ( reason ) ) ; }
catch ( Exception ex )
{
_logger . LogWarning ( ex , "OpcUaClientDriver '{DriverInstanceId}': a ConnectionLost subscriber faulted." ,
_driverInstanceId ) ;
}
}
/// <summary>
@@ -2771,14 +2994,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// frozen (sticky until the session is released), and the outage published once, so the host's watchdog
/// re-initialises and re-subscribes.
/// </summary>
private void DeclareSubscriptionLost ( string reason , DateTime now )
/// <param name="reason">Why, for the log and health.</param>
/// <param name="now">Detection time.</param>
/// <param name="serverReported">True when the SERVER said the subscription is gone (BadNoSubscription /
/// BadSubscriptionIdInvalid) — definitive, so <see cref="ConnectionLost"/> is raised. False for the publishing-stall
/// heuristic, an inference from silence that stays with the #527 watchdog (review of 3b88056f).</param>
private void DeclareSubscriptionLost ( string reason , DateTime now , bool serverReported )
{
if ( Interlocked . Exchange ( ref _subscriptionLost , 1 ) = = 1 ) return ;
UpdateHealth ( h = > new DriverHealth ( DriverState . Degraded , h . LastSuccessfulRead , $"subscription lost: {reason}" ) ) ;
_logger . LogWarning (
"OpcUaClientDriver '{DriverInstanceId}': {Reason}; subscribed values are BadNoCommunication until the " +
"host's comms-loss watchdog re-initialises the driver and re-subscribes." , _driverInstanceId , reason ) ;
"host re-initialises the driver and re-subscribes." , _driverInstanceId , reason ) ;
if ( Volatile . Read ( ref _sessionLost ) = = 0 ) PublishCommsLoss ( now ) ;
// #676: a subscription the SERVER says is gone never comes back on this session — only the re-subscribe after a
// re-initialise recreates it — so ask the host now rather than after the watchdog's 90 s. The stall heuristic is
// an inference, not a server statement, and keeps going through the watchdog.
if ( serverReported ) RaiseConnectionLost ( $"subscription lost: {reason}" ) ;
}
/// <summary>
@@ -2801,7 +3033,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
else
return ;
if ( tracked )
DeclareSubscriptionLost ( $"server reported {code} for a tracked subscription" , Clock . GetUtcNow ( ) . UtcDateTime ) ;
DeclareSubscriptionLost ( $"server reported {code} for a tracked subscription" , Clock . GetUtcNow ( ) . UtcDateTime ,
serverReported : true ) ;
}
/// <summary>
@@ -2811,18 +3044,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// </summary>
private void PublishCommsLoss ( DateTime now )
{
foreach ( var rs in _subscriptions . Values )
// Under the delivery lock (re-review of 30a9d5f0) so the Bad cannot interleave with a recovery publish.
lock ( _deliveryLock )
{
foreach ( var pair in rs . ItemHandlers )
foreach ( var rs in _subscriptions . Values )
{
var fullRef = RefOf ( pair . Item ) ;
_lastValues . TryGetValue ( fullRef , out var last ) ;
_awaitingRecovery [ fullRef ] = 0 ;
// F1: stamped with the DETECTION time, as the host's own comms-loss fan does. The recorder historizes
// SourceTimestampUtc ?? ServerTimestampUtc, so the last Good's source time here put the Bad at the
// Good row's (TagId, SourceTimeUtc) key, where the insert-if-absent historian dropped it.
RaiseDataChange ( new DataChangeEventArgs ( rs . Handle , fullRef , new DataValueSnapshot (
last ? . Value , StatusBadNoCommunication , now , now ) ) ) ;
foreach ( var pair in rs . ItemHandlers )
{
var fullRef = RefOf ( pair . Item ) ;
_lastValues . TryGetValue ( fullRef , out var last ) ;
_awaitingRecovery [ fullRef ] = 0 ;
_deliverySeq . AddOrUpdate ( fullRef , 1 , ( _ , n ) = > n + 1 ) ;
// F1: stamped with the DETECTION time, as the host's own comms-loss fan does. The recorder historizes
// SourceTimestampUtc ?? ServerTimestampUtc, so the last Good's source time here put the Bad at the
// Good row's (TagId, SourceTimeUtc) key, where the insert-if-absent historian dropped it.
RaiseDataChange ( new DataChangeEventArgs ( rs . Handle , fullRef , new DataValueSnapshot (
last ? . Value , StatusBadNoCommunication , now , now ) ) ) ;
}
}
}
AnnotateAlarmQuality ( StatusBadNoCommunication , now ) ;
@@ -2856,15 +3094,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
for ( var i = 0 ; i < refs . Count ; i + + )
{
if ( ! Current ( ) ) return ;
if ( ! _awaitingRecovery . TryRemove ( refs [ i ] , out _ ) ) continue ; // a notification got there firs t
_lastValues [ refs [ i ] ] = results [ i ] ;
RecordUpstreamGood ( refs [ i ] , results [ i ] ) ;
// The read is upstream data published inside the replay window: a replay older than it is late too.
if ( results [ i ] . SourceTimestampUtc is { } readAt & & StatusCode . IsGood ( results [ i ] . StatusCode ) )
_replayGuard . AddOrUpdate ( refs [ i ] , readAt . Ticks , ( _ , prior ) = > Math . Max ( prior , readAt . Ticks ) ) ;
foreach ( var rs in _subscriptions . Values )
if ( rs . ItemHandlers . Any ( p = > RefOf ( p . Item ) = = refs [ i ] ) )
RaiseDataChange ( new DataChangeEventArgs ( rs . Handle , refs [ i ] , results [ i ] ) ) ;
// One critical section with the live notification path (_deliveryLock): a notification tha t
// removed the ref first wins, and none can overtake this publish once it has decided.
lock ( _deliveryLock )
{
if ( ! _awaitingRecovery . TryRemove ( refs [ i ] , out _ ) ) continue ; // a notification got there first
PublishUpstreamLocked ( refs [ i ] , results [ i ] ) ;
}
}
}
@@ -2880,6 +3116,108 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
}
/// <summary>
/// Publish an upstream value read (not notified) for <paramref name="fullRef"/> to every subscription carrying it,
/// recording it as the last delivered value and — when Good with a source time — as the replay guard's mark, so a
/// replay older than the read is late too. Caller holds <see cref="_deliveryLock"/>.
/// </summary>
private void PublishUpstreamLocked ( string fullRef , DataValueSnapshot value )
{
_lastValues [ fullRef ] = value ;
_deliverySeq . AddOrUpdate ( fullRef , 1 , ( _ , n ) = > n + 1 ) ;
RecordUpstreamGood ( fullRef , value ) ;
if ( value . SourceTimestampUtc is { } readAt & & StatusCode . IsGood ( value . StatusCode )
& & Interlocked . Read ( ref _replayGuardUntilTicks ) ! = 0 )
_replayGuard . AddOrUpdate ( fullRef , readAt . Ticks , ( _ , prior ) = > Math . Max ( prior , readAt . Ticks ) ) ;
foreach ( var rs in _subscriptions . Values )
if ( rs . ItemHandlers . Any ( p = > RefOf ( p . Item ) = = fullRef ) )
RaiseDataChange ( new DataChangeEventArgs ( rs . Handle , fullRef , value ) ) ;
}
/// <summary>
/// #676 — the trailing reconcile read after an in-place recovery. Found by the TCP-stall wire test: a value change
/// the server published just after a stall ended rode a Publish response that had been in flight across the
/// stall; the SDK timed that request out (<c>BadTimeout</c>), discarded the response, and never republished the
/// message (the following keep-alives advanced the sequence number past it). The node stayed on the value from
/// the recovery read — stale, indefinitely, for a value that does not change again. Once every request that was
/// in flight across the stall has completed or timed out (<see cref="PostRecoveryReconcileDelay"/>), every
/// subscribed ref is read once more and published when <see cref="IsReconcileRepair"/> says the read differs from
/// what was last delivered and is not older than it. The decision and the publish are one critical section with
/// the live notification path (<see cref="_deliveryLock"/>), so a newer notification can never be overtaken.
/// Bound to the session's work token: a re-initialise or shutdown cancels it.
/// </summary>
/// <param name="session">The session that recovered; nothing happens if it is no longer current or is in outage.</param>
/// <param name="recovery">The recovery read, which runs first so this read is compared against its result.</param>
/// <param name="delay">How long to wait before reading (on <see cref="Clock"/>).</param>
/// <param name="ct">The session's background-work token.</param>
private async Task ReconcileAfterRecoveryAsync ( ISession session , Task recovery , TimeSpan delay , CancellationToken ct )
{
bool Current ( ) = > ! ct . IsCancellationRequested
& & ReferenceEquals ( session , Session )
& & Volatile . Read ( ref _sessionLost ) = = 0
& & Volatile . Read ( ref _subscriptionLost ) = = 0 ;
try
{
if ( delay > TimeSpan . Zero ) await Task . Delay ( delay , Clock , ct ) . ConfigureAwait ( false ) ;
await recovery . WaitAsync ( ct ) . ConfigureAwait ( false ) ;
if ( ! Current ( ) ) return ;
var refs = _subscriptions . Values
. SelectMany ( rs = > rs . ItemHandlers . Select ( p = > RefOf ( p . Item ) ) )
. Distinct ( StringComparer . Ordinal )
. ToList ( ) ;
if ( refs . Count = = 0 ) return ;
// Captured BEFORE the read is issued: a ref delivered to while the read is in flight has a value at least as
// new as anything the read could have sampled, so the read must not overwrite it.
var seqAtIssue = refs . Select ( r = > _deliverySeq . GetValueOrDefault ( r ) ) . ToArray ( ) ;
var ( results , transportOk ) = await ReadCoreAsync ( refs , ct , session ) . ConfigureAwait ( false ) ;
if ( ! transportOk ) return ; // a new outage — the keep-alive reports it
var published = 0 ;
for ( var i = 0 ; i < refs . Count ; i + + )
{
if ( ! Current ( ) ) return ;
lock ( _deliveryLock )
{
if ( _deliverySeq . GetValueOrDefault ( refs [ i ] ) ! = seqAtIssue [ i ] ) continue ;
if ( ! IsReconcileRepair ( refs [ i ] , results [ i ] ) ) continue ;
BeforeReconcilePublishForTest ? . Invoke ( refs [ i ] ) ;
_awaitingRecovery . TryRemove ( refs [ i ] , out _ ) ;
PublishUpstreamLocked ( refs [ i ] , results [ i ] ) ;
}
published + + ;
}
if ( published > 0 )
_logger . LogInformation (
"OpcUaClientDriver '{DriverInstanceId}': trailing re-read after recovery found {Count} of {Total} refs " +
"whose latest change never arrived through the subscription; published them." ,
_driverInstanceId , published , refs . Count ) ;
}
catch ( OperationCanceledException ) when ( ct . IsCancellationRequested ) { /* session released */ }
catch ( Exception ex )
{
_logger . LogWarning ( ex , "OpcUaClientDriver '{DriverInstanceId}': trailing re-read after a recovery failed." ,
_driverInstanceId ) ;
}
}
/// <summary>
/// Whether the trailing read's <paramref name="read"/> repairs <paramref name="fullRef"/>: its (value, status)
/// differs from the last value delivered for the ref, and it is not OLDER than it. Any status counts — a Good→Bad
/// or Good→Uncertain change lost in the stalled pipeline is as stale as a lost value. "Older" is judged on source
/// timestamps only when both carry one; a read with no source timestamp (a server that does not send them) is the
/// server's CURRENT state, read after the stalled pipeline drained, so it counts as newer. An EQUAL source
/// timestamp with different content is published too: it is not a replay but the server's current state, and it
/// repairs the one change the replay guard drops by design (a status-only change that kept its source time inside
/// the window). Caller holds <see cref="_deliveryLock"/>.
/// </summary>
private bool IsReconcileRepair ( string fullRef , DataValueSnapshot read )
{
if ( ! _lastValues . TryGetValue ( fullRef , out var last ) ) return true ;
if ( last . StatusCode = = read . StatusCode & & Utils . IsEqual ( last . Value , read . Value ) ) return false ;
return read . SourceTimestampUtc is not { } src
| | last . SourceTimestampUtc is not { } lastSrc
| | src > = lastSrc ;
}
/// <summary>Quality-only annotation (#626 pattern) of every subscribed alarm condition.</summary>
private void AnnotateAlarmQuality ( uint quality , DateTime now )
{
@@ -2981,6 +3319,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
catch { /* timeout or already disposed — proceed */ }
_gate . Dispose ( ) ;
try { _sessionWorkCts . Cancel ( ) ; } catch ( ObjectDisposedException ) { }
SdkTelemetry . Release ( _sdkTelemetry ) ;
}
@@ -3010,6 +3349,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
catch { /* timeout or already disposed */ }
_gate . Dispose ( ) ;
try { _sessionWorkCts . Cancel ( ) ; } catch ( ObjectDisposedException ) { }
SdkTelemetry . Release ( _sdkTelemetry ) ;
}
}
Joseph Doherty