feat(opcuaclient): #676 follow-ups — endpoint rewrite, stall wire test, fast definitive-loss recovery
- Endpoint rewrite: the driver now connects to the configured host AND port (scheme match, advertised path kept) instead of the server-advertised URL — fixes NAT/port-map upstreams and lets an in-test proxy carry the session (SDK SelectEndpoint rule). - TcpStallProxy + OpcUaClientDriverStallWireTests: the in-place blip (#670) is now wire-tested on loopback — one Bad, frozen read stamp, same session, identical value republished Good, a post-recovery change arrives, no source-time regression after the SDK's late backlog flush. - Defect found by that test, fixed: a change published right after a stall rode a Publish response the SDK timed out and never republished, leaving the node Good on the pre-change value indefinitely. New trailing reconcile read (Timeout + KeepAliveInterval after recovery) publishes any Good value newer than the last accepted one, through the replay guard's mark. - IDriverConnectionLossSource (Core.Abstractions): opt-in definitive-loss signal. DriverInstanceActor attaches in PreStart / detaches in PostStop and turns a raise into a disconnect only while Connected. OpcUaClient raises once per session on keep-alive BadSessionIdInvalid/BadSessionClosed/BadSessionNotActivated/ BadServerHalted (measured: the SDK never re-opens a halted channel) or a dropped subscription; ambiguous codes stay with the #527 watchdog. - Replay guard: one Information summary per recovery when the window closes ("dropped N late notifications across M refs"), with a timer so quiet subscriptions log it too. - ReconnectPeriod deleted (options, AdminUI form, docs, tests); stored configs that carry reconnectPeriod still parse in the factory, probe and browser. - Docs: OpcUaClient.md, DriverLifecycle.md, driver-specs.md, CLAUDE.md.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,31 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Optional driver capability (Gitea #676) — a driver that can tell, from its protocol, that its connection is
|
||||
/// DEFINITIVELY gone (it can never recover in place, only a re-initialise brings it back) raises
|
||||
/// <see cref="ConnectionLost"/> so the host re-initialises it at once instead of waiting out the #527 comms-loss
|
||||
/// watchdog (~90 s of frozen <see cref="DriverHealth.LastSuccessfulRead"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Definitive only.</b> An AMBIGUOUS failure — a timeout, a stalled socket, "no communication" — must NOT
|
||||
/// raise: the connection may yet resume in place (a <c>docker pause</c>, a congested link), and tearing it
|
||||
/// down would cost the session, its subscriptions and the in-place recovery. Ambiguous failures keep going
|
||||
/// through the watchdog, whose conjunction (non-Healthy AND a frozen read stamp) is the honest test for them.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The host (<c>DriverInstanceActor</c>) acts on a raise only while the driver is Connected — in any other
|
||||
/// state a re-initialise is already pending or running — and it subscribes for its own lifetime, not per
|
||||
/// connect, because the <see cref="IDriver"/> instance can outlive the actor. A driver should raise at most
|
||||
/// once per connection and re-arm when it is re-initialised.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IDriverConnectionLossSource
|
||||
{
|
||||
/// <summary>Raised (on the driver's own thread) when the driver's connection is definitively lost.</summary>
|
||||
event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
|
||||
}
|
||||
|
||||
/// <summary>Event payload for <see cref="IDriverConnectionLossSource.ConnectionLost"/>.</summary>
|
||||
/// <param name="Reason">Driver-supplied reason for the diagnostic log (e.g. the protocol status that proved the loss).</param>
|
||||
public sealed record DriverConnectionLossEventArgs(string Reason);
|
||||
@@ -101,12 +101,6 @@ public sealed record OpcUaClientDriverOptions
|
||||
/// <summary>Client-side keep-alive interval.</summary>
|
||||
public TimeSpan KeepAliveInterval { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Retained for config compatibility only — no longer used (Gitea #670). The driver has no in-driver reconnect
|
||||
/// handler; a lost session is recovered by the host's comms-loss watchdog re-initialising the driver.
|
||||
/// </summary>
|
||||
public TimeSpan ReconnectPeriod { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// When <c>true</c>, the driver accepts any self-signed / untrusted server certificate.
|
||||
/// Dev-only — must be <c>false</c> in production so MITM attacks against the opc.tcp
|
||||
|
||||
@@ -29,7 +29,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
|
||||
/// back on its own recovers in place.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDisposable, IAsyncDisposable
|
||||
public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDriverConnectionLossSource, IDisposable, IAsyncDisposable
|
||||
{
|
||||
/// <summary>Authored tags whose TagConfig carried no usable <c>nodeId</c>, so there is no upstream node
|
||||
/// to dial (Gitea #541). This driver's skip path was the quietest of the twelve: <c>NamespaceMap</c>
|
||||
@@ -83,6 +83,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
_logger = logger ?? NullLogger<OpcUaClientDriver>.Instance;
|
||||
_secretResolver = secretResolver ?? NullSecretResolver.Instance;
|
||||
_sdkTelemetry = logger is null ? SdkTelemetry.Context : SdkTelemetry.ForLogger(logger);
|
||||
PostRecoveryReconcileDelay = () => _options.Timeout + _options.KeepAliveInterval;
|
||||
}
|
||||
|
||||
/// <summary>Mutable so <see cref="InitializeAsync"/> can adopt a re-parsed config on reinitialize
|
||||
@@ -135,6 +136,12 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>Occurs when the host connectivity status of the remote OPC UA server changes.</summary>
|
||||
public event EventHandler<HostStatusChangedEventArgs>? OnHostStatusChanged;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Gitea #676 — raised at most once per session, and only on a DEFINITIVE loss
|
||||
/// (<see cref="IsDefinitiveSessionLoss"/>, or a tracked subscription the server dropped). Re-armed when the session
|
||||
/// is released (re-initialise / shutdown).</remarks>
|
||||
public event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
|
||||
|
||||
// OPC UA StatusCode constants the driver surfaces for local-side faults. Upstream-server
|
||||
// StatusCodes are passed through verbatim per driver-specs.md §8 "cascading quality" —
|
||||
// downstream clients need to distinguish 'remote source down' from 'local driver failure'.
|
||||
@@ -164,6 +171,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>Test-only seam (#670): the most recent in-place recovery task.</summary>
|
||||
internal Task RecoveryTaskForTest => _recoveryTask;
|
||||
|
||||
/// <summary>Test seam (#676): the trailing reconcile read scheduled by the last in-place recovery.</summary>
|
||||
internal Task ReconcileTaskForTest => _reconcileTask;
|
||||
|
||||
/// <summary>Test-only seam (#670 review M1): raise the session's PublishError as the SDK publish loop does.</summary>
|
||||
/// <param name="status">The publish error status.</param>
|
||||
/// <param name="subscriptionId">The subscription id the error names (0 = none).</param>
|
||||
@@ -282,6 +292,25 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>The last in-place recovery started (#670). Test seam: lets a test await it deterministically.</summary>
|
||||
private Task _recoveryTask = Task.CompletedTask;
|
||||
|
||||
/// <summary>#676 — 1 once <see cref="ConnectionLost"/> has been raised for the CURRENT session (once per session);
|
||||
/// cleared when the session is released, so a re-initialised driver reports its next loss.</summary>
|
||||
private int _connectionLostRaised;
|
||||
|
||||
/// <summary>#676 — the trailing reconcile read scheduled by the last in-place recovery. Test seam.</summary>
|
||||
private Task _reconcileTask = Task.CompletedTask;
|
||||
|
||||
/// <summary>#676 — late notifications the replay guard dropped since it was armed, and the refs they were for;
|
||||
/// summarised in ONE Information line when the window disarms. Guarded by <see cref="_replayGuardLock"/>.</summary>
|
||||
private int _replayDrops;
|
||||
private readonly HashSet<string> _replayDropRefs = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Test seam (#676): how long after an in-place recovery the trailing reconcile read runs. Production: the
|
||||
/// operation timeout plus one keep-alive interval — by then every request that was in flight across the stall
|
||||
/// (including the Publish responses whose notifications the SDK can lose) has completed or timed out.
|
||||
/// </summary>
|
||||
internal Func<TimeSpan> PostRecoveryReconcileDelay { get; set; }
|
||||
|
||||
/// <summary>Test seam (#670 review M1): whether a subscription has stopped publishing. Production asks the SDK.</summary>
|
||||
internal Func<Subscription, bool> IsPublishingStopped { get; set; } = static s => s.Created && s.PublishingStopped;
|
||||
|
||||
@@ -883,9 +912,34 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
$"No endpoint at '{endpointUrl}' matches SecurityPolicy={policy} + SecurityMode={mode}. " +
|
||||
$"Server advertises: {advertised}");
|
||||
}
|
||||
// #676: connect where the operator pointed us, not where the server says it lives.
|
||||
match.EndpointUrl = RewriteEndpointUrl(match.EndpointUrl, endpointUrl);
|
||||
return match;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #676 — the address to open the session on. A server reports its endpoints with ITS view of its own
|
||||
/// address; behind NAT, a port map, a container port publish or a proxy that address is unreachable from here
|
||||
/// (or bypasses the proxy). When the advertised URL's scheme matches the configured one, the configured host
|
||||
/// AND port replace the advertised ones and the advertised path is kept — the SDK's own
|
||||
/// <c>CoreClientUtils.SelectEndpoint</c> rule, which this driver's policy-aware selection bypassed. A different
|
||||
/// scheme is a different transport and is left alone; an unparseable advertised URL falls back to the
|
||||
/// configured one.
|
||||
/// </summary>
|
||||
/// <param name="advertisedUrl">The <c>EndpointUrl</c> the server returned from GetEndpoints.</param>
|
||||
/// <param name="configuredUrl">The URL the driver was configured with (the one GetEndpoints was sent to).</param>
|
||||
/// <returns>The URL to connect the session to.</returns>
|
||||
internal static string RewriteEndpointUrl(string? advertisedUrl, string configuredUrl)
|
||||
{
|
||||
if (!Uri.TryCreate(configuredUrl, UriKind.Absolute, out var configured)) return advertisedUrl ?? configuredUrl;
|
||||
if (string.IsNullOrWhiteSpace(advertisedUrl)
|
||||
|| !Uri.TryCreate(advertisedUrl, UriKind.Absolute, out var advertised))
|
||||
return configuredUrl;
|
||||
if (!string.Equals(advertised.Scheme, configured.Scheme, StringComparison.OrdinalIgnoreCase)) return advertisedUrl;
|
||||
var builder = new UriBuilder(advertised) { Host = configured.DnsSafeHost, Port = configured.Port };
|
||||
return builder.Uri.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Build a <see cref="UserIdentity"/> carrying a client user-authentication
|
||||
/// certificate loaded from <see cref="OpcUaClientDriverOptions.UserCertificatePath"/>.
|
||||
@@ -986,6 +1040,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
Interlocked.Exchange(ref _sessionLost, 0);
|
||||
Interlocked.Exchange(ref _subscriptionLost, 0);
|
||||
Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
|
||||
// #676: re-arm the definitive-loss signal for the next session.
|
||||
Interlocked.Exchange(ref _connectionLostRaised, 0);
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on the best-effort wire calls a teardown makes against a possibly-dead upstream.</summary>
|
||||
@@ -1021,8 +1077,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
// previous connection's value bookkeeping applies to them.
|
||||
_lastValues.Clear();
|
||||
_lastGoodSourceTicks.Clear();
|
||||
_replayGuard.Clear();
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
_replayGuard.Clear();
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
_replayDrops = 0;
|
||||
_replayDropRefs.Clear();
|
||||
}
|
||||
_awaitingRecovery.Clear();
|
||||
// #619 re-review: the last-told alarm states (and ack targets) deliberately SURVIVE a shutdown. A
|
||||
// ReinitializeAsync (comms-loss watchdog / Force Reconnect) is followed by the host re-subscribing the
|
||||
@@ -1766,6 +1827,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
if (IsLateReplay(fullRef, snapshot.SourceTimestampUtc, out var newest))
|
||||
{
|
||||
_diag.Record(OpcUaClientDiagnostics.StaleNotificationDropped);
|
||||
// #676: the OTel counter and the Debug line below are invisible on a default-configured node; the
|
||||
// per-recovery Information summary logged at disarm is what makes the guard observable.
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
_replayDrops++;
|
||||
_replayDropRefs.Add(fullRef);
|
||||
}
|
||||
_logger.LogDebug(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
|
||||
"(source {Source:O} is not newer than {Newest:O}).", _driverInstanceId, fullRef, snapshot.SourceTimestampUtc,
|
||||
@@ -1798,14 +1866,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
if (until == 0) return false;
|
||||
if (Clock.GetUtcNow().UtcDateTime.Ticks >= until)
|
||||
{
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
if (Interlocked.Read(ref _replayGuardUntilTicks) == until)
|
||||
{
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
_replayGuard.Clear();
|
||||
}
|
||||
}
|
||||
DisarmReplayGuard(until);
|
||||
return false;
|
||||
}
|
||||
if (source is not { } src) return false;
|
||||
@@ -1821,11 +1882,36 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expire the replay guard armed until <paramref name="until"/> — a no-op when it was re-armed or already disarmed
|
||||
/// since — and log ONE Information summary of what it dropped (#676). The guard's OTel counter and per-drop Debug
|
||||
/// line are invisible on a default-configured node, so before this a recovery's drops left no trace at all.
|
||||
/// </summary>
|
||||
/// <param name="until">The expiry the caller observed; only that arming is disarmed.</param>
|
||||
private void DisarmReplayGuard(long until)
|
||||
{
|
||||
int drops, refs;
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
if (Interlocked.Read(ref _replayGuardUntilTicks) != until) return;
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
_replayGuard.Clear();
|
||||
drops = _replayDrops;
|
||||
refs = _replayDropRefs.Count;
|
||||
_replayDrops = 0;
|
||||
_replayDropRefs.Clear();
|
||||
}
|
||||
_logger.LogInformation(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': post-recovery replay window closed — dropped {Drops} late " +
|
||||
"notifications across {Refs} refs after recovery.", _driverInstanceId, drops, refs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #670 F2 — arm the replay guard on a same-session recovery: every ref with a known Good upstream source time,
|
||||
/// for max(30 s, 2 × the longest tracked subscription keep-alive window).
|
||||
/// </summary>
|
||||
private void ArmReplayGuard(DateTime now)
|
||||
/// <returns>The window's expiry in clock ticks.</returns>
|
||||
private long ArmReplayGuard(DateTime now)
|
||||
{
|
||||
var window = ReplayGuardMinimum;
|
||||
foreach (var sub in _subscriptions.Values.Select(r => r.Subscription))
|
||||
@@ -1839,6 +1925,25 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
foreach (var kv in _lastGoodSourceTicks) _replayGuard[kv.Key] = kv.Value;
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, (now + window).Ticks);
|
||||
}
|
||||
return (now + window).Ticks;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #676 — disarm the replay guard (and log its summary) when its window ends, even if no notification arrives to
|
||||
/// disarm it lazily in <see cref="IsLateReplay"/> — a quiet subscription would otherwise never log the summary.
|
||||
/// </summary>
|
||||
private async Task DisarmReplayGuardWhenDueAsync(long until)
|
||||
{
|
||||
try
|
||||
{
|
||||
var wait = new DateTime(until, DateTimeKind.Utc) - Clock.GetUtcNow().UtcDateTime;
|
||||
if (wait > TimeSpan.Zero) await Task.Delay(wait + TimeSpan.FromMilliseconds(100)).ConfigureAwait(false);
|
||||
if (Clock.GetUtcNow().UtcDateTime.Ticks >= until) DisarmReplayGuard(until);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "OpcUaClientDriver '{DriverInstanceId}': replay-guard disarm timer faulted.", _driverInstanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2683,7 +2788,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// session-recreate path cloned subscriptions the driver never re-tracked, and it kept health Healthy while
|
||||
/// it did so — the #670 symptom. A session that does not come back keeps health Degraded with the read stamp
|
||||
/// frozen, which is exactly what the host's #527 comms-loss watchdog trips on: ShutdownAsync + InitializeAsync
|
||||
/// + re-subscribe, whose initial values come back Good.
|
||||
/// + re-subscribe, whose initial values come back Good. When the keep-alive status proves the session is gone
|
||||
/// for good (<see cref="IsDefinitiveSessionLoss"/>) the driver also raises <see cref="ConnectionLost"/> (#676),
|
||||
/// so the host runs that sequence at once instead of after the watchdog's silence window.
|
||||
/// </remarks>
|
||||
private void OnKeepAlive(ISession sender, KeepAliveEventArgs e)
|
||||
{
|
||||
@@ -2706,12 +2813,15 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
UpdateHealth(_ => new DriverHealth(DriverState.Healthy, now, null));
|
||||
// A fresh grace: every subscription reports stopped right after an outage until its next message.
|
||||
Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
|
||||
ArmReplayGuard(now);
|
||||
var guardUntil = ArmReplayGuard(now);
|
||||
_diag.Record(OpcUaClientDiagnostics.KeepAliveRecovered);
|
||||
_logger.LogInformation(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': upstream keep-alive recovered on the same session; " +
|
||||
"re-reading the refs published Bad during the outage.", _driverInstanceId);
|
||||
_recoveryTask = Task.Run(() => RecoverAfterOutageAsync(sender));
|
||||
var recovery = Task.Run(() => RecoverAfterOutageAsync(sender));
|
||||
_recoveryTask = recovery;
|
||||
_reconcileTask = ReconcileAfterRecoveryAsync(sender, recovery, PostRecoveryReconcileDelay());
|
||||
_ = DisarmReplayGuardWhenDueAsync(guardUntil);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2729,14 +2839,56 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
TransitionTo(HostState.Stopped);
|
||||
_diag.Record(OpcUaClientDiagnostics.KeepAliveMiss);
|
||||
UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"keep-alive: {e.Status}"));
|
||||
if (Interlocked.Exchange(ref _sessionLost, 1) == 1) return;
|
||||
if (Volatile.Read(ref _subscriptionLost) == 1) return; // already published Bad for this connection
|
||||
// #676: the first bad keep-alive is often ambiguous and a definitive one follows, so this is checked on EVERY
|
||||
// bad keep-alive — but only after the outage below has been published, so the Bad reaches subscribers first.
|
||||
var keepAliveCode = e.Status?.StatusCode ?? StatusCodes.Good;
|
||||
var definitive = IsDefinitiveSessionLoss(keepAliveCode);
|
||||
// A subscription loss already published Bad for this connection.
|
||||
if (Interlocked.Exchange(ref _sessionLost, 1) == 0 && Volatile.Read(ref _subscriptionLost) == 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
|
||||
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
|
||||
_driverInstanceId, e.Status);
|
||||
PublishCommsLoss(now);
|
||||
}
|
||||
if (definitive)
|
||||
RaiseConnectionLost($"upstream session keep-alive reported {StatusCodes.GetBrowseName(keepAliveCode.Code)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #676 — keep-alive statuses that prove the session can NEVER recover in place, so waiting out the #527
|
||||
/// watchdog would only delay the inevitable re-initialise. The server forgot the session
|
||||
/// (<c>BadSessionIdInvalid</c> — what a re-opened channel to a restarted server answers), closed it
|
||||
/// (<c>BadSessionClosed</c>) or never activated it (<c>BadSessionNotActivated</c>); or the server HALTED
|
||||
/// (<c>BadServerHalted</c>) — measured on the #676 restart wire test, the SDK never re-opens a channel the
|
||||
/// server halted: 15 s after the server was back the same session still failed every request with
|
||||
/// <c>BadServerHalted</c>, so a halted session is dead for good. Everything else — <c>BadNoCommunication</c>,
|
||||
/// <c>BadConnectionClosed</c>, <c>BadNotConnected</c>, request timeouts — is AMBIGUOUS (a stalled link that
|
||||
/// resumes keeps its session, see the stall wire test) and stays with the watchdog.
|
||||
/// </summary>
|
||||
/// <param name="status">A keep-alive status.</param>
|
||||
/// <returns>True when the session is definitively gone.</returns>
|
||||
internal static bool IsDefinitiveSessionLoss(StatusCode status) =>
|
||||
status == StatusCodes.BadSessionIdInvalid
|
||||
|| status == StatusCodes.BadSessionClosed
|
||||
|| status == StatusCodes.BadSessionNotActivated
|
||||
|| status == StatusCodes.BadServerHalted;
|
||||
|
||||
/// <summary>#676 — raise <see cref="ConnectionLost"/> once for the current session, isolating a faulting subscriber.</summary>
|
||||
/// <param name="reason">Why the connection is definitively gone, for the host's log.</param>
|
||||
private void RaiseConnectionLost(string reason)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _connectionLostRaised, 1) == 1) return;
|
||||
_logger.LogWarning(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
|
||||
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
|
||||
_driverInstanceId, e.Status);
|
||||
PublishCommsLoss(now);
|
||||
"OpcUaClientDriver '{DriverInstanceId}': connection definitively lost ({Reason}); asking the host to " +
|
||||
"re-initialise now instead of waiting for the comms-loss watchdog.", _driverInstanceId, reason);
|
||||
try { ConnectionLost?.Invoke(this, new DriverConnectionLossEventArgs(reason)); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': a ConnectionLost subscriber faulted.",
|
||||
_driverInstanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2777,8 +2929,11 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"subscription lost: {reason}"));
|
||||
_logger.LogWarning(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': {Reason}; subscribed values are BadNoCommunication until the " +
|
||||
"host's comms-loss watchdog re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
|
||||
"host re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
|
||||
if (Volatile.Read(ref _sessionLost) == 0) PublishCommsLoss(now);
|
||||
// #676: a dead subscription never comes back on this session — only the re-subscribe after a re-initialise
|
||||
// recreates it — so this is definitive: ask the host now rather than after the watchdog's 90 s.
|
||||
RaiseConnectionLost($"subscription lost: {reason}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2880,6 +3035,74 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #676 — the trailing reconcile read after an in-place recovery. Found by the TCP-stall wire test: a value change
|
||||
/// the server published just after a stall ended rode a Publish response that had been in flight across the
|
||||
/// stall; the SDK timed that request out (<c>BadTimeout</c>), discarded the response, and never republished the
|
||||
/// message (the following keep-alives advanced the sequence number past it). The node stayed Good on the value
|
||||
/// from the recovery read — stale, indefinitely, for a value that does not change again. Once every request that
|
||||
/// was in flight across the stall has completed or timed out (<see cref="PostRecoveryReconcileDelay"/>), every
|
||||
/// subscribed ref is read once more and any Good value NEWER than the last one accepted for that ref is published
|
||||
/// (through the replay guard's mark while its window is open, so it can never regress a node).
|
||||
/// </summary>
|
||||
/// <param name="session">The session that recovered; nothing happens if it is no longer current or is in outage.</param>
|
||||
/// <param name="recovery">The recovery read, which runs first so this read is compared against its result.</param>
|
||||
/// <param name="delay">How long to wait before reading.</param>
|
||||
private async Task ReconcileAfterRecoveryAsync(ISession session, Task recovery, TimeSpan delay)
|
||||
{
|
||||
bool Current() => ReferenceEquals(session, Session)
|
||||
&& Volatile.Read(ref _sessionLost) == 0
|
||||
&& Volatile.Read(ref _subscriptionLost) == 0;
|
||||
try
|
||||
{
|
||||
if (delay > TimeSpan.Zero) await Task.Delay(delay).ConfigureAwait(false);
|
||||
await recovery.ConfigureAwait(false);
|
||||
if (!Current()) return;
|
||||
var refs = _subscriptions.Values
|
||||
.SelectMany(rs => rs.ItemHandlers.Select(p => RefOf(p.Item)))
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
if (refs.Count == 0) return;
|
||||
var (results, transportOk) = await ReadCoreAsync(refs, CancellationToken.None, session).ConfigureAwait(false);
|
||||
if (!transportOk) return; // a new outage — the keep-alive reports it
|
||||
var published = 0;
|
||||
for (var i = 0; i < refs.Count; i++)
|
||||
{
|
||||
if (!Current()) return;
|
||||
var r = results[i];
|
||||
if (!StatusCode.IsGood(r.StatusCode) || r.SourceTimestampUtc is not { } src) continue;
|
||||
if (!IsNewerThanAccepted(refs[i], src)) continue;
|
||||
RecordUpstreamGood(refs[i], r);
|
||||
_lastValues[refs[i]] = r;
|
||||
_awaitingRecovery.TryRemove(refs[i], out _);
|
||||
foreach (var rs in _subscriptions.Values)
|
||||
if (rs.ItemHandlers.Any(p => RefOf(p.Item) == refs[i]))
|
||||
RaiseDataChange(new DataChangeEventArgs(rs.Handle, refs[i], r));
|
||||
published++;
|
||||
}
|
||||
if (published > 0)
|
||||
_logger.LogInformation(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': trailing re-read after recovery found {Count} of {Total} refs " +
|
||||
"whose latest change never arrived through the subscription; published them.",
|
||||
_driverInstanceId, published, refs.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': trailing re-read after a recovery failed.",
|
||||
_driverInstanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>#676 — whether a Good upstream value with source time <paramref name="source"/> is newer than the last one
|
||||
/// accepted for <paramref name="fullRef"/>. Inside the replay window this IS the guard's check (and advances its mark);
|
||||
/// outside it, a strict comparison against the last accepted Good source time.</summary>
|
||||
private bool IsNewerThanAccepted(string fullRef, DateTime source)
|
||||
{
|
||||
var until = Interlocked.Read(ref _replayGuardUntilTicks);
|
||||
if (until != 0 && Clock.GetUtcNow().UtcDateTime.Ticks < until) return !IsLateReplay(fullRef, source, out _);
|
||||
return !_lastGoodSourceTicks.TryGetValue(fullRef, out var last) || source.Ticks > last;
|
||||
}
|
||||
|
||||
/// <summary>Quality-only annotation (#626 pattern) of every subscribed alarm condition.</summary>
|
||||
private void AnnotateAlarmQuality(uint quality, DateTime now)
|
||||
{
|
||||
|
||||
-8
@@ -53,11 +53,6 @@
|
||||
<InputNumber @bind-Value="_form.OpcUa.KeepAliveIntervalSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Default 5 s.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Reconnect period (s)</label>
|
||||
<InputNumber @bind-Value="_form.OpcUa.ReconnectPeriodSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Initial delay after session drop. Default 5 s.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Max discovered nodes</label>
|
||||
<InputNumber @bind-Value="_form.OpcUa.MaxDiscoveredNodes" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
@@ -330,7 +325,6 @@
|
||||
public int TimeoutSeconds { get; set; } = 10;
|
||||
public int SessionTimeoutSeconds { get; set; } = 120;
|
||||
public int KeepAliveIntervalSeconds { get; set; } = 5;
|
||||
public int ReconnectPeriodSeconds { get; set; } = 5;
|
||||
public int MaxDiscoveredNodes { get; set; } = 10_000;
|
||||
public int MaxBrowseDepth { get; set; } = 10;
|
||||
|
||||
@@ -370,7 +364,6 @@
|
||||
TimeoutSeconds = (int)r.Timeout.TotalSeconds,
|
||||
SessionTimeoutSeconds = (int)r.SessionTimeout.TotalSeconds,
|
||||
KeepAliveIntervalSeconds = (int)r.KeepAliveInterval.TotalSeconds,
|
||||
ReconnectPeriodSeconds = (int)r.ReconnectPeriod.TotalSeconds,
|
||||
MaxDiscoveredNodes = r.MaxDiscoveredNodes,
|
||||
MaxBrowseDepth = r.MaxBrowseDepth,
|
||||
SecurityMode = r.SecurityMode,
|
||||
@@ -404,7 +397,6 @@
|
||||
Timeout = TimeSpan.FromSeconds(TimeoutSeconds),
|
||||
SessionTimeout = TimeSpan.FromSeconds(SessionTimeoutSeconds),
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(KeepAliveIntervalSeconds),
|
||||
ReconnectPeriod = TimeSpan.FromSeconds(ReconnectPeriodSeconds),
|
||||
MaxDiscoveredNodes = MaxDiscoveredNodes,
|
||||
MaxBrowseDepth = MaxBrowseDepth,
|
||||
SecurityMode = SecurityMode,
|
||||
|
||||
@@ -153,6 +153,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// behaviour for the same reason as <see cref="RediscoveryRaised"/> — a probe tick can land while the
|
||||
/// driver is between connects.</summary>
|
||||
private sealed record HostStatusRaised;
|
||||
/// <summary>Self-sent when the wrapped driver raises <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676)
|
||||
/// — it KNOWS its connection is gone for good. Marshals the event off the driver's thread. Acted on only in
|
||||
/// Connected (it becomes a disconnect at once, instead of after the #527 watchdog's silence window); every other
|
||||
/// behaviour is already reconnecting or connecting and ignores it.</summary>
|
||||
private sealed record ConnectionLossRaised(DriverConnectionLossEventArgs Args);
|
||||
|
||||
public sealed class RetryConnect
|
||||
{
|
||||
@@ -233,6 +238,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
private EventHandler<AlarmEventArgs>? _alarmEventHandler;
|
||||
private EventHandler<RediscoveryEventArgs>? _rediscoveryHandler;
|
||||
private EventHandler<HostStatusChangedEventArgs>? _hostStatusHandler;
|
||||
private EventHandler<DriverConnectionLossEventArgs>? _connectionLossHandler;
|
||||
|
||||
/// <summary>When the driver last raised <see cref="IRediscoverable.OnRediscoveryNeeded"/>, and the reason
|
||||
/// it gave. Null until the first raise. Carried on every subsequent health snapshot so the AdminUI can
|
||||
@@ -400,6 +406,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// has no connection affinity, and a driver can observe a remote change while disconnected.
|
||||
AttachRediscoverySource();
|
||||
AttachHostStatusSource();
|
||||
AttachConnectionLossSource();
|
||||
PublishHealthSnapshot();
|
||||
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
|
||||
}
|
||||
@@ -427,6 +434,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// Stubbed drivers have no upstream alarm system — swallow the ack (it's fire-and-forget, no reply).
|
||||
Receive<RouteAlarmAck>(_ => { /* stubbed drivers have no alarm backend */ });
|
||||
Receive<DisconnectObserved>(_ => { /* stubbed drivers don't disconnect */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
Receive<ForceReconnect>(_ => { /* stubbed drivers don't reconnect */ });
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
// Stubbed drivers never enter Connected, so they never kick discovery; swallow defensively in case a
|
||||
@@ -468,6 +476,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
});
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already connecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
|
||||
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
|
||||
Receive<SubscriptionEstablished>(msg =>
|
||||
@@ -500,15 +509,12 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// reconnects but never manages a read is measured from THIS connect, not from a stale earlier one.
|
||||
_connectedSinceUtc = DateTime.UtcNow;
|
||||
|
||||
Receive<DisconnectObserved>(msg =>
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting",
|
||||
_driverInstanceId, msg.Reason);
|
||||
DetachSubscription();
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
});
|
||||
Receive<DisconnectObserved>(msg => HandleDisconnectObserved(msg.Reason));
|
||||
// #676 — the driver KNOWS its connection is gone (e.g. OpcUaClient: the upstream server halted or forgot the
|
||||
// session). Same handling as the watchdog's disconnect, minus the ~90 s of silence it would need to prove it.
|
||||
// Handled inline rather than re-told as DisconnectObserved, so it cannot land in Reconnecting as an unhandled
|
||||
// message.
|
||||
Receive<ConnectionLossRaised>(msg => HandleDisconnectObserved($"driver reported connection lost: {msg.Args.Reason}"));
|
||||
Receive<ForceReconnect>(_ =>
|
||||
{
|
||||
_log.Info("DriverInstance {Id}: ForceReconnect requested by admin; re-entering Reconnecting", _driverInstanceId);
|
||||
@@ -764,6 +770,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<InitializeFailed>(_ => { /* keep retrying via timer */ });
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already reconnecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
|
||||
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
|
||||
Receive<SubscriptionEstablished>(msg =>
|
||||
@@ -1105,6 +1112,50 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// one), marshaling each transition to the actor thread. Idempotent; mirrors
|
||||
/// <see cref="AttachRediscoverySource"/>, including the PreStart-not-per-connect placement — a probe
|
||||
/// loop can report a host down while the driver itself is between connects.</summary>
|
||||
/// <summary>
|
||||
/// Subscribe the driver's <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676), if it is one,
|
||||
/// marshalling each raise to the actor thread — actor state is never touched on the driver's thread. Attached
|
||||
/// once in <c>PreStart</c> and detached in <c>PostStop</c>, like <see cref="AttachRediscoverySource"/>: the
|
||||
/// <see cref="IDriver"/> instance can outlive this actor, so a per-connect attach would leak a handler (holding
|
||||
/// a dead <c>Self</c>) per respawn.
|
||||
/// </summary>
|
||||
private void AttachConnectionLossSource()
|
||||
{
|
||||
if (_driver is not IDriverConnectionLossSource src || _connectionLossHandler is not null) return;
|
||||
var self = Self;
|
||||
_connectionLossHandler = (_, e) => self.Tell(new ConnectionLossRaised(e));
|
||||
src.ConnectionLost += _connectionLossHandler;
|
||||
}
|
||||
|
||||
/// <summary>Symmetric teardown, called from PostStop — same leak argument as <see cref="DetachRediscoverySource"/>.</summary>
|
||||
private void DetachConnectionLossSource()
|
||||
{
|
||||
if (_driver is IDriverConnectionLossSource src && _connectionLossHandler is not null)
|
||||
src.ConnectionLost -= _connectionLossHandler;
|
||||
_connectionLossHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A connection loss reported while NOT Connected (#676): a (re)initialise is already pending or running, so
|
||||
/// there is nothing to tear down — and acting on it could tear down the connection that attempt is building.
|
||||
/// </summary>
|
||||
private void IgnoreConnectionLoss(ConnectionLossRaised msg) =>
|
||||
_log.Debug("DriverInstance {Id}: driver reported connection lost ({Reason}) while not connected — ignored",
|
||||
_driverInstanceId, msg.Args.Reason);
|
||||
|
||||
/// <summary>
|
||||
/// Connected → Reconnecting on an observed disconnect — from the #527 watchdog or a driver's definitive
|
||||
/// <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676).
|
||||
/// </summary>
|
||||
private void HandleDisconnectObserved(string reason)
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting", _driverInstanceId, reason);
|
||||
DetachSubscription();
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
}
|
||||
|
||||
private void AttachHostStatusSource()
|
||||
{
|
||||
if (_driver is not IHostConnectivityProbe src || _hostStatusHandler is not null) return;
|
||||
@@ -1497,6 +1548,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
|
||||
DetachRediscoverySource();
|
||||
DetachHostStatusSource();
|
||||
DetachConnectionLossSource();
|
||||
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
|
||||
Reference in New Issue
Block a user