dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge feat/676-opcuaclient-followups: endpoint host+port rewrite, TCP-stall wire test, ReconnectPeriod removed, IDriverConnectionLossSource fast recovery, trailing re-read, replay summary (#676)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-11 11:45:26 -04:00
parent 029a916288 27bf4826d6
commit 50c09e21d5
22 changed files with 1876 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs
-6
View File
@@ -101,12 +101,6 @@ public sealed record OpcUaClientDriverOptions
/// <summary>Client-side keep-alive interval.</summary>
public TimeSpan KeepAliveInterval { get; init; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Retained for config compatibility only — no longer used (Gitea #670). The driver has no in-driver reconnect
/// handler; a lost session is recovered by the host's comms-loss watchdog re-initialising the driver.
/// </summary>
public TimeSpan ReconnectPeriod { get; init; } = TimeSpan.FromSeconds(5);
/// <summary>
/// When <c>true</c>, the driver accepts any self-signed / untrusted server certificate.
/// Dev-only — must be <c>false</c> in production so MITM attacks against the opc.tcp
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDriver.cs
+396 -56
View File
@@ -29,7 +29,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
/// back on its own recovers in place.
/// </para>
/// </remarks>
public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDisposable, IAsyncDisposable
public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDriverConnectionLossSource, IDisposable, IAsyncDisposable
{
/// <summary>Authored tags whose TagConfig carried no usable <c>nodeId</c>, so there is no upstream node
/// to dial (Gitea #541). This driver's skip path was the quietest of the twelve: <c>NamespaceMap</c>
@@ -83,6 +83,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
_logger = logger ?? NullLogger<OpcUaClientDriver>.Instance;
_secretResolver = secretResolver ?? NullSecretResolver.Instance;
_sdkTelemetry = logger is null ? SdkTelemetry.Context : SdkTelemetry.ForLogger(logger);
PostRecoveryReconcileDelay = () => _options.Timeout + _options.KeepAliveInterval;
}
/// <summary>Mutable so <see cref="InitializeAsync"/> can adopt a re-parsed config on reinitialize
@@ -135,6 +136,14 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>Occurs when the host connectivity status of the remote OPC UA server changes.</summary>
public event EventHandler<HostStatusChangedEventArgs>? OnHostStatusChanged;
/// <inheritdoc />
/// <remarks>Gitea #676 — raised at most once per session, and only on a DEFINITIVE loss
/// (<see cref="IsDefinitiveSessionLoss"/>, or a tracked subscription the SERVER reported dropped — never the
/// publishing-stall heuristic). Re-armed when the session is released (re-initialise / shutdown). Known gap: the host
/// acts on it only while Connected, so a raise it receives before it has processed the successful initialise is
/// swallowed and that session falls back to the #527 watchdog.</remarks>
public event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
// OPC UA StatusCode constants the driver surfaces for local-side faults. Upstream-server
// StatusCodes are passed through verbatim per driver-specs.md §8 "cascading quality" —
// downstream clients need to distinguish 'remote source down' from 'local driver failure'.
@@ -164,6 +173,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>Test-only seam (#670): the most recent in-place recovery task.</summary>
internal Task RecoveryTaskForTest => _recoveryTask;
/// <summary>Test seam (#676): the trailing reconcile read scheduled by the last in-place recovery.</summary>
internal Task ReconcileTaskForTest => _reconcileTask;
/// <summary>Test-only seam (#670 review M1): raise the session's PublishError as the SDK publish loop does.</summary>
/// <param name="status">The publish error status.</param>
/// <param name="subscriptionId">The subscription id the error names (0 = none).</param>
@@ -282,6 +294,25 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>The last in-place recovery started (#670). Test seam: lets a test await it deterministically.</summary>
private Task _recoveryTask = Task.CompletedTask;
/// <summary>#676 — 1 once <see cref="ConnectionLost"/> has been raised for the CURRENT session (once per session);
/// cleared when the session is released, so a re-initialised driver reports its next loss.</summary>
private int _connectionLostRaised;
/// <summary>#676 — the trailing reconcile read scheduled by the last in-place recovery. Test seam.</summary>
private Task _reconcileTask = Task.CompletedTask;
/// <summary>#676 — late notifications the replay guard dropped since it was armed, and the refs they were for;
/// summarised in ONE Information line when the window disarms. Guarded by <see cref="_replayGuardLock"/>.</summary>
private int _replayDrops;
private readonly HashSet<string> _replayDropRefs = new(StringComparer.Ordinal);
/// <summary>
/// Test seam (#676): how long after an in-place recovery the trailing reconcile read runs. Production: the
/// operation timeout plus one keep-alive interval — by then every request that was in flight across the stall
/// (including the Publish responses whose notifications the SDK can lose) has completed or timed out.
/// </summary>
internal Func<TimeSpan> PostRecoveryReconcileDelay { get; set; }
/// <summary>Test seam (#670 review M1): whether a subscription has stopped publishing. Production asks the SDK.</summary>
internal Func<Subscription, bool> IsPublishingStopped { get; set; } = static s => s.Created && s.PublishingStopped;
@@ -326,6 +357,37 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// <summary>Serialises arming against expiry, so an expiry racing a re-arm cannot wipe the new seeds.</summary>
private readonly object _replayGuardLock = new();
/// <summary>
/// Review of 3b88056f — serialises "decide whether this upstream value may be published" with "publish it" across
/// the three paths that publish upstream data: live notifications, the recovery read and the trailing reconcile
/// read. Without it a path could pass its newer-than check, lose the CPU to another path publishing a NEWER value,
/// and then deliver its older one last — regressing the node. It also covers the comms-loss Bad fan
/// (<see cref="PublishCommsLoss"/>). One driver-wide lock rather than one per ref: it also serialises delivery
/// ACROSS subscriptions — with several subscriptions the SDK dispatches each subscription's messages on its own
/// worker, so their notifications no longer run in parallel. That is the cost, and it is small: each critical
/// section is a dictionary update plus a subscriber call that only <c>Tell</c>s an actor (tens of nanoseconds to
/// a few microseconds), nothing awaits under it, and per-subscription message order is unaffected. Because
/// subscribers are invoked under it, a subscriber must never block on, or call back into, this driver.
/// </summary>
private readonly Lock _deliveryLock = new();
/// <summary>
/// Re-review of 30a9d5f0 — per RawPath, a counter bumped (under <see cref="_deliveryLock"/>) every time a value is
/// delivered for it. The trailing re-read captures it before issuing its read and skips any ref delivered to since:
/// without source timestamps the read's value cannot be ordered against a notification that landed while the read
/// was in flight, and publishing it would regress the node to the value the read sampled.
/// </summary>
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, long> _deliverySeq =
new(StringComparer.Ordinal);
/// <summary>Review of 3b88056f — cancels the post-recovery background work (trailing read, replay-guard disarm timer)
/// of the current session; cancelled and replaced whenever the session is released, and on dispose.</summary>
private CancellationTokenSource _sessionWorkCts = new();
/// <summary>Test seam (review of 3b88056f): invoked inside the trailing read's critical section, after it decided
/// to publish a ref and before it does — where a racing notification would otherwise overtake it.</summary>
internal Action<string>? BeforeReconcilePublishForTest { get; set; }
/// <summary>Floor of the replay-guard window; the live gate saw the late replay 4–5 s after recovery.</summary>
private static readonly TimeSpan ReplayGuardMinimum = TimeSpan.FromSeconds(30);
@@ -883,9 +945,40 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
$"No endpoint at '{endpointUrl}' matches SecurityPolicy={policy} + SecurityMode={mode}. " +
$"Server advertises: {advertised}");
}
// #676: connect where the operator pointed us, not where the server says it lives.
match.EndpointUrl = RewriteEndpointUrl(match.EndpointUrl, endpointUrl);
return match;
}
/// <summary>
/// Gitea #676 — the address to open the session on. A server reports its endpoints with ITS view of its own
/// address; behind NAT, a port map, a container port publish or a proxy that address is unreachable from here
/// (or bypasses the proxy). When the advertised URL's scheme matches the configured one, the configured host
/// AND port (when it names one) replace the advertised ones and the advertised path is kept — the SDK's own
/// <c>CoreClientUtils.SelectEndpoint</c> rule, which this driver's policy-aware selection bypassed. A different
/// scheme is a different transport and is left alone; an unparseable advertised URL falls back to the
/// configured one.
/// </summary>
/// <param name="advertisedUrl">The <c>EndpointUrl</c> the server returned from GetEndpoints.</param>
/// <param name="configuredUrl">The URL the driver was configured with (the one GetEndpoints was sent to).</param>
/// <returns>The URL to connect the session to.</returns>
internal static string RewriteEndpointUrl(string? advertisedUrl, string configuredUrl)
{
if (!Uri.TryCreate(configuredUrl, UriKind.Absolute, out var configured)) return advertisedUrl ?? configuredUrl;
if (string.IsNullOrWhiteSpace(advertisedUrl)
|| !Uri.TryCreate(advertisedUrl, UriKind.Absolute, out var advertised))
return configuredUrl;
if (!string.Equals(advertised.Scheme, configured.Scheme, StringComparison.OrdinalIgnoreCase)) return advertisedUrl;
// A port-less configured URL (Uri.Port == -1 for opc.tcp) keeps the advertised port — copying -1 would drop it
// and the SDK would fall back to 4840 (review of 3b88056f; same fix as the client's EndpointSelector).
var builder = new UriBuilder(advertised)
{
Host = configured.DnsSafeHost,
Port = configured.Port > 0 ? configured.Port : advertised.Port,
};
return builder.Uri.ToString();
}
/// <summary>
/// Build a <see cref="UserIdentity"/> carrying a client user-authentication
/// certificate loaded from <see cref="OpcUaClientDriverOptions.UserCertificatePath"/>.
@@ -986,6 +1079,20 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
Interlocked.Exchange(ref _sessionLost, 0);
Interlocked.Exchange(ref _subscriptionLost, 0);
Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
// #676: re-arm the definitive-loss signal for the next session.
Interlocked.Exchange(ref _connectionLostRaised, 0);
// Review of 3b88056f: the released session's post-recovery timers must not outlive it.
CancelSessionWork();
}
/// <summary>Cancel the current session's background work and start a fresh token for the next one.</summary>
private void CancelSessionWork()
{
// Cancelled but deliberately NOT disposed (re-review of 30a9d5f0): the keep-alive thread may be reading the old
// source's Token concurrently, and a disposed source throws ObjectDisposedException there. Nothing here is
// timer-backed (no CancelAfter), so an undisposed source holds no unmanaged resource.
var old = Interlocked.Exchange(ref _sessionWorkCts, new CancellationTokenSource());
old.Cancel();
}
/// <summary>Upper bound on the best-effort wire calls a teardown makes against a possibly-dead upstream.</summary>
@@ -1021,8 +1128,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
// previous connection's value bookkeeping applies to them.
_lastValues.Clear();
_lastGoodSourceTicks.Clear();
_replayGuard.Clear();
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
lock (_replayGuardLock)
{
_replayGuard.Clear();
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
_replayDrops = 0;
_replayDropRefs.Clear();
}
_awaitingRecovery.Clear();
// #619 re-review: the last-told alarm states (and ack targets) deliberately SURVIVE a shutdown. A
// ReinitializeAsync (comms-loss watchdog / Force Reconnect) is followed by the host re-subscribing the
@@ -1762,20 +1874,33 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
StatusCode: dv.StatusCode.Code,
SourceTimestampUtc: dv.SourceTimestamp == DateTime.MinValue ? null : dv.SourceTimestamp,
ServerTimestampUtc: dv.ServerTimestamp == DateTime.MinValue ? DateTime.UtcNow : dv.ServerTimestamp);
// F2: inside the post-recovery replay window only, a late OLDER notification must not regress the node.
if (IsLateReplay(fullRef, snapshot.SourceTimestampUtc, out var newest))
// Review of 3b88056f: the replay check and the publish are one critical section shared with the recovery and
// trailing reads, so an older value can never be delivered after a newer one (see _deliveryLock).
lock (_deliveryLock)
{
_diag.Record(OpcUaClientDiagnostics.StaleNotificationDropped);
_logger.LogDebug(
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
"(source {Source:O} is not newer than {Newest:O}).", _driverInstanceId, fullRef, snapshot.SourceTimestampUtc,
new DateTime(newest, DateTimeKind.Utc));
return;
// F2: inside the post-recovery replay window only, a late OLDER notification must not regress the node.
if (IsLateReplay(fullRef, snapshot.SourceTimestampUtc, out var newest))
{
_diag.Record(OpcUaClientDiagnostics.StaleNotificationDropped);
// #676: the OTel counter and the Debug line below are invisible on a default-configured node; the
// per-recovery Information summary logged at disarm is what makes the guard observable.
lock (_replayGuardLock)
{
_replayDrops++;
_replayDropRefs.Add(fullRef);
}
_logger.LogDebug(
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
"(source {Source:O} is not newer than {Newest:O}).", _driverInstanceId, fullRef, snapshot.SourceTimestampUtc,
new DateTime(newest, DateTimeKind.Utc));
return;
}
RecordUpstreamGood(fullRef, snapshot);
_lastValues[fullRef] = snapshot;
_deliverySeq.AddOrUpdate(fullRef, 1, (_, n) => n + 1);
_awaitingRecovery.TryRemove(fullRef, out _);
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullRef, snapshot));
}
RecordUpstreamGood(fullRef, snapshot);
_lastValues[fullRef] = snapshot;
_awaitingRecovery.TryRemove(fullRef, out _);
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullRef, snapshot));
}
/// <summary>#670 F2 — remember the last accepted Good upstream source time for <paramref name="fullRef"/>.</summary>
@@ -1798,14 +1923,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
if (until == 0) return false;
if (Clock.GetUtcNow().UtcDateTime.Ticks >= until)
{
lock (_replayGuardLock)
{
if (Interlocked.Read(ref _replayGuardUntilTicks) == until)
{
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
_replayGuard.Clear();
}
}
DisarmReplayGuard(until);
return false;
}
if (source is not { } src) return false;
@@ -1821,11 +1939,40 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
}
/// <summary>
/// Expire the replay guard armed until <paramref name="until"/> — a no-op when it was re-armed or already disarmed
/// since — and log ONE Information summary of what it dropped (#676). The guard's OTel counter and per-drop Debug
/// line are invisible on a default-configured node, so before this a recovery's drops left no trace at all.
/// </summary>
/// <param name="until">The expiry the caller observed; only that arming is disarmed.</param>
private void DisarmReplayGuard(long until)
{
int drops, refs;
lock (_replayGuardLock)
{
if (Interlocked.Read(ref _replayGuardUntilTicks) != until) return;
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
_replayGuard.Clear();
drops = _replayDrops;
refs = _replayDropRefs.Count;
_replayDrops = 0;
_replayDropRefs.Clear();
}
LogReplaySummary(drops, refs);
}
/// <summary>One line per replay window: Information when the guard dropped something, Debug otherwise.</summary>
private void LogReplaySummary(int drops, int refs) =>
_logger.Log(drops > 0 ? LogLevel.Information : LogLevel.Debug,
"OpcUaClientDriver '{DriverInstanceId}': post-recovery replay window closed — dropped {Drops} late " +
"notifications across {Refs} refs after recovery.", _driverInstanceId, drops, refs);
/// <summary>
/// #670 F2 — arm the replay guard on a same-session recovery: every ref with a known Good upstream source time,
/// for max(30 s, 2 × the longest tracked subscription keep-alive window).
/// </summary>
private void ArmReplayGuard(DateTime now)
/// <returns>The window's expiry in clock ticks.</returns>
private long ArmReplayGuard(DateTime now)
{
var window = ReplayGuardMinimum;
foreach (var sub in _subscriptions.Values.Select(r => r.Subscription))
@@ -1833,12 +1980,39 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
var keepAliveWindow = TimeSpan.FromMilliseconds(sub.CurrentPublishingInterval * (double)sub.CurrentKeepAliveCount);
if (keepAliveWindow * 2 > window) window = keepAliveWindow * 2;
}
int supersededDrops, supersededRefs;
lock (_replayGuardLock)
{
// A re-arm inside an open window ends that window: report what it dropped and count afresh.
supersededDrops = _replayDrops;
supersededRefs = _replayDropRefs.Count;
_replayDrops = 0;
_replayDropRefs.Clear();
_replayGuard.Clear();
foreach (var kv in _lastGoodSourceTicks) _replayGuard[kv.Key] = kv.Value;
Interlocked.Exchange(ref _replayGuardUntilTicks, (now + window).Ticks);
}
if (supersededDrops > 0) LogReplaySummary(supersededDrops, supersededRefs);
return (now + window).Ticks;
}
/// <summary>
/// #676 — disarm the replay guard (and log its summary) when its window ends, even if no notification arrives to
/// disarm it lazily in <see cref="IsLateReplay"/> — a quiet subscription would otherwise never log the summary.
/// </summary>
private async Task DisarmReplayGuardWhenDueAsync(long until, CancellationToken ct)
{
try
{
var wait = new DateTime(until, DateTimeKind.Utc) - Clock.GetUtcNow().UtcDateTime;
if (wait > TimeSpan.Zero) await Task.Delay(wait + TimeSpan.FromMilliseconds(100), Clock, ct).ConfigureAwait(false);
if (Clock.GetUtcNow().UtcDateTime.Ticks >= until) DisarmReplayGuard(until);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { /* session released */ }
catch (Exception ex)
{
_logger.LogDebug(ex, "OpcUaClientDriver '{DriverInstanceId}': replay-guard disarm timer faulted.", _driverInstanceId);
}
}
/// <summary>
@@ -2683,7 +2857,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// session-recreate path cloned subscriptions the driver never re-tracked, and it kept health Healthy while
/// it did so — the #670 symptom. A session that does not come back keeps health Degraded with the read stamp
/// frozen, which is exactly what the host's #527 comms-loss watchdog trips on: ShutdownAsync + InitializeAsync
/// + re-subscribe, whose initial values come back Good.
/// + re-subscribe, whose initial values come back Good. When the keep-alive status proves the session is gone
/// for good (<see cref="IsDefinitiveSessionLoss"/>) the driver also raises <see cref="ConnectionLost"/> (#676),
/// so the host runs that sequence at once instead of after the watchdog's silence window.
/// </remarks>
private void OnKeepAlive(ISession sender, KeepAliveEventArgs e)
{
@@ -2706,18 +2882,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
UpdateHealth(_ => new DriverHealth(DriverState.Healthy, now, null));
// A fresh grace: every subscription reports stopped right after an outage until its next message.
Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
ArmReplayGuard(now);
var guardUntil = ArmReplayGuard(now);
_diag.Record(OpcUaClientDiagnostics.KeepAliveRecovered);
_logger.LogInformation(
"OpcUaClientDriver '{DriverInstanceId}': upstream keep-alive recovered on the same session; " +
"re-reading the refs published Bad during the outage.", _driverInstanceId);
_recoveryTask = Task.Run(() => RecoverAfterOutageAsync(sender));
var recovery = Task.Run(() => RecoverAfterOutageAsync(sender));
_recoveryTask = recovery;
var sessionWork = _sessionWorkCts.Token;
_reconcileTask = ReconcileAfterRecoveryAsync(sender, recovery, PostRecoveryReconcileDelay(), sessionWork);
_ = DisarmReplayGuardWhenDueAsync(guardUntil, sessionWork);
return;
}
if (FindStalledSubscription(now) is { } stalled)
{
DeclareSubscriptionLost($"subscription '{stalled}' stopped publishing while the session stayed up", now);
DeclareSubscriptionLost($"subscription '{stalled}' stopped publishing while the session stayed up", now,
serverReported: false);
return;
}
@@ -2729,14 +2910,56 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
TransitionTo(HostState.Stopped);
_diag.Record(OpcUaClientDiagnostics.KeepAliveMiss);
UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"keep-alive: {e.Status}"));
if (Interlocked.Exchange(ref _sessionLost, 1) == 1) return;
if (Volatile.Read(ref _subscriptionLost) == 1) return; // already published Bad for this connection
// #676: the first bad keep-alive is often ambiguous and a definitive one follows, so this is checked on EVERY
// bad keep-alive — but only after the outage below has been published, so the Bad reaches subscribers first.
var keepAliveCode = e.Status?.StatusCode ?? StatusCodes.Good;
var definitive = IsDefinitiveSessionLoss(keepAliveCode);
// A subscription loss already published Bad for this connection.
if (Interlocked.Exchange(ref _sessionLost, 1) == 0 && Volatile.Read(ref _subscriptionLost) == 0)
{
_logger.LogWarning(
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
_driverInstanceId, e.Status);
PublishCommsLoss(now);
}
if (definitive)
RaiseConnectionLost($"upstream session keep-alive reported {StatusCodes.GetBrowseName(keepAliveCode.Code)}");
}
/// <summary>
/// #676 — keep-alive statuses that prove the session can NEVER recover in place, so waiting out the #527
/// watchdog would only delay the inevitable re-initialise. The server forgot the session
/// (<c>BadSessionIdInvalid</c> — what a re-opened channel to a restarted server answers), closed it
/// (<c>BadSessionClosed</c>) or never activated it (<c>BadSessionNotActivated</c>); or the server HALTED
/// (<c>BadServerHalted</c>) — measured on the #676 restart wire test, the SDK never re-opens a channel the
/// server halted: 15 s after the server was back the same session still failed every request with
/// <c>BadServerHalted</c>, so a halted session is dead for good. Everything else — <c>BadNoCommunication</c>,
/// <c>BadConnectionClosed</c>, <c>BadNotConnected</c>, request timeouts — is AMBIGUOUS (a stalled link that
/// resumes keeps its session, see the stall wire test) and stays with the watchdog.
/// </summary>
/// <param name="status">A keep-alive status.</param>
/// <returns>True when the session is definitively gone.</returns>
internal static bool IsDefinitiveSessionLoss(StatusCode status) =>
status == StatusCodes.BadSessionIdInvalid
|| status == StatusCodes.BadSessionClosed
|| status == StatusCodes.BadSessionNotActivated
|| status == StatusCodes.BadServerHalted;
/// <summary>#676 — raise <see cref="ConnectionLost"/> once for the current session, isolating a faulting subscriber.</summary>
/// <param name="reason">Why the connection is definitively gone, for the host's log.</param>
private void RaiseConnectionLost(string reason)
{
if (Interlocked.Exchange(ref _connectionLostRaised, 1) == 1) return;
_logger.LogWarning(
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
_driverInstanceId, e.Status);
PublishCommsLoss(now);
"OpcUaClientDriver '{DriverInstanceId}': connection definitively lost ({Reason}); asking the host to " +
"re-initialise now instead of waiting for the comms-loss watchdog.", _driverInstanceId, reason);
try { ConnectionLost?.Invoke(this, new DriverConnectionLossEventArgs(reason)); }
catch (Exception ex)
{
_logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': a ConnectionLost subscriber faulted.",
_driverInstanceId);
}
}
/// <summary>
@@ -2771,14 +2994,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// frozen (sticky until the session is released), and the outage published once, so the host's watchdog
/// re-initialises and re-subscribes.
/// </summary>
private void DeclareSubscriptionLost(string reason, DateTime now)
/// <param name="reason">Why, for the log and health.</param>
/// <param name="now">Detection time.</param>
/// <param name="serverReported">True when the SERVER said the subscription is gone (BadNoSubscription /
/// BadSubscriptionIdInvalid) — definitive, so <see cref="ConnectionLost"/> is raised. False for the publishing-stall
/// heuristic, an inference from silence that stays with the #527 watchdog (review of 3b88056f).</param>
private void DeclareSubscriptionLost(string reason, DateTime now, bool serverReported)
{
if (Interlocked.Exchange(ref _subscriptionLost, 1) == 1) return;
UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"subscription lost: {reason}"));
_logger.LogWarning(
"OpcUaClientDriver '{DriverInstanceId}': {Reason}; subscribed values are BadNoCommunication until the " +
"host's comms-loss watchdog re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
"host re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
if (Volatile.Read(ref _sessionLost) == 0) PublishCommsLoss(now);
// #676: a subscription the SERVER says is gone never comes back on this session — only the re-subscribe after a
// re-initialise recreates it — so ask the host now rather than after the watchdog's 90 s. The stall heuristic is
// an inference, not a server statement, and keeps going through the watchdog.
if (serverReported) RaiseConnectionLost($"subscription lost: {reason}");
}
/// <summary>
@@ -2801,7 +3033,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
else
return;
if (tracked)
DeclareSubscriptionLost($"server reported {code} for a tracked subscription", Clock.GetUtcNow().UtcDateTime);
DeclareSubscriptionLost($"server reported {code} for a tracked subscription", Clock.GetUtcNow().UtcDateTime,
serverReported: true);
}
/// <summary>
@@ -2811,18 +3044,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
/// </summary>
private void PublishCommsLoss(DateTime now)
{
foreach (var rs in _subscriptions.Values)
// Under the delivery lock (re-review of 30a9d5f0) so the Bad cannot interleave with a recovery publish.
lock (_deliveryLock)
{
foreach (var pair in rs.ItemHandlers)
foreach (var rs in _subscriptions.Values)
{
var fullRef = RefOf(pair.Item);
_lastValues.TryGetValue(fullRef, out var last);
_awaitingRecovery[fullRef] = 0;
// F1: stamped with the DETECTION time, as the host's own comms-loss fan does. The recorder historizes
// SourceTimestampUtc ?? ServerTimestampUtc, so the last Good's source time here put the Bad at the
// Good row's (TagId, SourceTimeUtc) key, where the insert-if-absent historian dropped it.
RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, new DataValueSnapshot(
last?.Value, StatusBadNoCommunication, now, now)));
foreach (var pair in rs.ItemHandlers)
{
var fullRef = RefOf(pair.Item);
_lastValues.TryGetValue(fullRef, out var last);
_awaitingRecovery[fullRef] = 0;
_deliverySeq.AddOrUpdate(fullRef, 1, (_, n) => n + 1);
// F1: stamped with the DETECTION time, as the host's own comms-loss fan does. The recorder historizes
// SourceTimestampUtc ?? ServerTimestampUtc, so the last Good's source time here put the Bad at the
// Good row's (TagId, SourceTimeUtc) key, where the insert-if-absent historian dropped it.
RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, new DataValueSnapshot(
last?.Value, StatusBadNoCommunication, now, now)));
}
}
}
AnnotateAlarmQuality(StatusBadNoCommunication, now);
@@ -2856,15 +3094,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
for (var i = 0; i < refs.Count; i++)
{
if (!Current()) return;
if (!_awaitingRecovery.TryRemove(refs[i], out _)) continue; // a notification got there first
_lastValues[refs[i]] = results[i];
RecordUpstreamGood(refs[i], results[i]);
// The read is upstream data published inside the replay window: a replay older than it is late too.
if (results[i].SourceTimestampUtc is { } readAt && StatusCode.IsGood(results[i].StatusCode))
_replayGuard.AddOrUpdate(refs[i], readAt.Ticks, (_, prior) => Math.Max(prior, readAt.Ticks));
foreach (var rs in _subscriptions.Values)
if (rs.ItemHandlers.Any(p => RefOf(p.Item) == refs[i]))
RaiseDataChange(new DataChangeEventArgs(rs.Handle, refs[i], results[i]));
// One critical section with the live notification path (_deliveryLock): a notification that
// removed the ref first wins, and none can overtake this publish once it has decided.
lock (_deliveryLock)
{
if (!_awaitingRecovery.TryRemove(refs[i], out _)) continue; // a notification got there first
PublishUpstreamLocked(refs[i], results[i]);
}
}
}
@@ -2880,6 +3116,108 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
}
/// <summary>
/// Publish an upstream value read (not notified) for <paramref name="fullRef"/> to every subscription carrying it,
/// recording it as the last delivered value and — when Good with a source time — as the replay guard's mark, so a
/// replay older than the read is late too. Caller holds <see cref="_deliveryLock"/>.
/// </summary>
private void PublishUpstreamLocked(string fullRef, DataValueSnapshot value)
{
_lastValues[fullRef] = value;
_deliverySeq.AddOrUpdate(fullRef, 1, (_, n) => n + 1);
RecordUpstreamGood(fullRef, value);
if (value.SourceTimestampUtc is { } readAt && StatusCode.IsGood(value.StatusCode)
&& Interlocked.Read(ref _replayGuardUntilTicks) != 0)
_replayGuard.AddOrUpdate(fullRef, readAt.Ticks, (_, prior) => Math.Max(prior, readAt.Ticks));
foreach (var rs in _subscriptions.Values)
if (rs.ItemHandlers.Any(p => RefOf(p.Item) == fullRef))
RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, value));
}
/// <summary>
/// #676 — the trailing reconcile read after an in-place recovery. Found by the TCP-stall wire test: a value change
/// the server published just after a stall ended rode a Publish response that had been in flight across the
/// stall; the SDK timed that request out (<c>BadTimeout</c>), discarded the response, and never republished the
/// message (the following keep-alives advanced the sequence number past it). The node stayed on the value from
/// the recovery read — stale, indefinitely, for a value that does not change again. Once every request that was
/// in flight across the stall has completed or timed out (<see cref="PostRecoveryReconcileDelay"/>), every
/// subscribed ref is read once more and published when <see cref="IsReconcileRepair"/> says the read differs from
/// what was last delivered and is not older than it. The decision and the publish are one critical section with
/// the live notification path (<see cref="_deliveryLock"/>), so a newer notification can never be overtaken.
/// Bound to the session's work token: a re-initialise or shutdown cancels it.
/// </summary>
/// <param name="session">The session that recovered; nothing happens if it is no longer current or is in outage.</param>
/// <param name="recovery">The recovery read, which runs first so this read is compared against its result.</param>
/// <param name="delay">How long to wait before reading (on <see cref="Clock"/>).</param>
/// <param name="ct">The session's background-work token.</param>
private async Task ReconcileAfterRecoveryAsync(ISession session, Task recovery, TimeSpan delay, CancellationToken ct)
{
bool Current() => !ct.IsCancellationRequested
&& ReferenceEquals(session, Session)
&& Volatile.Read(ref _sessionLost) == 0
&& Volatile.Read(ref _subscriptionLost) == 0;
try
{
if (delay > TimeSpan.Zero) await Task.Delay(delay, Clock, ct).ConfigureAwait(false);
await recovery.WaitAsync(ct).ConfigureAwait(false);
if (!Current()) return;
var refs = _subscriptions.Values
.SelectMany(rs => rs.ItemHandlers.Select(p => RefOf(p.Item)))
.Distinct(StringComparer.Ordinal)
.ToList();
if (refs.Count == 0) return;
// Captured BEFORE the read is issued: a ref delivered to while the read is in flight has a value at least as
// new as anything the read could have sampled, so the read must not overwrite it.
var seqAtIssue = refs.Select(r => _deliverySeq.GetValueOrDefault(r)).ToArray();
var (results, transportOk) = await ReadCoreAsync(refs, ct, session).ConfigureAwait(false);
if (!transportOk) return; // a new outage — the keep-alive reports it
var published = 0;
for (var i = 0; i < refs.Count; i++)
{
if (!Current()) return;
lock (_deliveryLock)
{
if (_deliverySeq.GetValueOrDefault(refs[i]) != seqAtIssue[i]) continue;
if (!IsReconcileRepair(refs[i], results[i])) continue;
BeforeReconcilePublishForTest?.Invoke(refs[i]);
_awaitingRecovery.TryRemove(refs[i], out _);
PublishUpstreamLocked(refs[i], results[i]);
}
published++;
}
if (published > 0)
_logger.LogInformation(
"OpcUaClientDriver '{DriverInstanceId}': trailing re-read after recovery found {Count} of {Total} refs " +
"whose latest change never arrived through the subscription; published them.",
_driverInstanceId, published, refs.Count);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { /* session released */ }
catch (Exception ex)
{
_logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': trailing re-read after a recovery failed.",
_driverInstanceId);
}
}
/// <summary>
/// Whether the trailing read's <paramref name="read"/> repairs <paramref name="fullRef"/>: its (value, status)
/// differs from the last value delivered for the ref, and it is not OLDER than it. Any status counts — a Good→Bad
/// or Good→Uncertain change lost in the stalled pipeline is as stale as a lost value. "Older" is judged on source
/// timestamps only when both carry one; a read with no source timestamp (a server that does not send them) is the
/// server's CURRENT state, read after the stalled pipeline drained, so it counts as newer. An EQUAL source
/// timestamp with different content is published too: it is not a replay but the server's current state, and it
/// repairs the one change the replay guard drops by design (a status-only change that kept its source time inside
/// the window). Caller holds <see cref="_deliveryLock"/>.
/// </summary>
private bool IsReconcileRepair(string fullRef, DataValueSnapshot read)
{
if (!_lastValues.TryGetValue(fullRef, out var last)) return true;
if (last.StatusCode == read.StatusCode && Utils.IsEqual(last.Value, read.Value)) return false;
return read.SourceTimestampUtc is not { } src
|| last.SourceTimestampUtc is not { } lastSrc
|| src >= lastSrc;
}
/// <summary>Quality-only annotation (#626 pattern) of every subscribed alarm condition.</summary>
private void AnnotateAlarmQuality(uint quality, DateTime now)
{
@@ -2981,6 +3319,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
catch { /* timeout or already disposed — proceed */ }
_gate.Dispose();
try { _sessionWorkCts.Cancel(); } catch (ObjectDisposedException) { }
SdkTelemetry.Release(_sdkTelemetry);
}
@@ -3010,6 +3349,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
}
catch { /* timeout or already disposed */ }
_gate.Dispose();
try { _sessionWorkCts.Cancel(); } catch (ObjectDisposedException) { }
SdkTelemetry.Release(_sdkTelemetry);
}
}