Merge fix/670-opcuaclient-session-loss: OpcUaClient upstream session loss goes Bad and recovers (#670)

src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs

@@ -80,11 +80,11 @@ public static class OtOpcUaTelemetry
     /// Session/publish-level events of an OPC UA Client driver instance (#276): the operator's view of a
     /// flapping upstream. Tagged <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) +
     /// <c>event</c> = <c>publish_response|notification|data_change|bad_status_value|publish_error|
-    /// keepalive_miss|reconnect_attempt|reconnect_success|reconnect_exhausted</c>.
+    /// keepalive_miss|keepalive_recovered</c>.
     /// </summary>
     public static readonly Counter<long> OpcUaClientEvents =
         Meter.CreateCounter<long>("otopcua.driver.opcuaclient.events", unit: "{event}",
-            description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|reconnect_attempt|reconnect_success|reconnect_exhausted).");
+            description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered).");
 
     // ---------------- VirtualTag / ScriptedAlarm engines ----------------
 

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs

@@ -101,7 +101,10 @@ public sealed record OpcUaClientDriverOptions
     /// <summary>Client-side keep-alive interval.</summary>
     public TimeSpan KeepAliveInterval { get; init; } = TimeSpan.FromSeconds(5);
 
-    /// <summary>Initial reconnect delay after a session drop.</summary>
+    /// <summary>
+    ///     Retained for config compatibility only — no longer used (Gitea #670). The driver has no in-driver reconnect
+    ///     handler; a lost session is recovered by the host's comms-loss watchdog re-initialising the driver.
+    /// </summary>
     public TimeSpan ReconnectPeriod { get; init; } = TimeSpan.FromSeconds(5);
 
     /// <summary>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDiagnostics.cs

@@ -22,12 +22,8 @@ internal sealed class OpcUaClientDiagnostics
     public const string PublishError = "publish_error";
     /// <summary>A keep-alive came back Bad.</summary>
     public const string KeepAliveMiss = "keepalive_miss";
-    /// <summary>The reconnect handler was armed.</summary>
-    public const string ReconnectAttempt = "reconnect_attempt";
-    /// <summary>A reconnect landed a new session.</summary>
-    public const string ReconnectSuccess = "reconnect_success";
-    /// <summary>A reconnect handler exhausted its retry window.</summary>
-    public const string ReconnectExhausted = "reconnect_exhausted";
+    /// <summary>A keep-alive came back Good after a bad one on the same session (an in-place recovery, #670).</summary>
+    public const string KeepAliveRecovered = "keepalive_recovered";
 
     private readonly string _driverInstanceId;
 

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDriver.cs

@@ -23,10 +23,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
 ///     </para>
 ///     <para>
 ///         <b>Session lifetime</b>: a single <see cref="Session"/> per driver instance.
-///         Subscriptions multiplex onto that session; SDK reconnect handler takes the session
-///         down and brings it back up on remote-server restart — the driver must re-send
-///         subscriptions + TransferSubscriptions on reconnect to avoid dangling
-///         monitored-item handles. That mechanic lands in PR 69.
+///         Subscriptions multiplex onto that session. A lost session is reported (health Degraded,
+///         subscribed values BadNoCommunication — Gitea #670) and recovered by the host's comms-loss
+///         watchdog re-initialising the driver and re-subscribing; a session whose keep-alive comes
+///         back on its own recovers in place.
 ///     </para>
 /// </remarks>
 public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDisposable, IAsyncDisposable
@@ -82,6 +82,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         _diag = new OpcUaClientDiagnostics(driverInstanceId);
         _logger = logger ?? NullLogger<OpcUaClientDriver>.Instance;
         _secretResolver = secretResolver ?? NullSecretResolver.Instance;
+        _sdkTelemetry = logger is null ? SdkTelemetry.Context : SdkTelemetry.ForLogger(logger);
     }
 
     /// <summary>Mutable so <see cref="InitializeAsync"/> can adopt a re-parsed config on reinitialize
@@ -140,6 +141,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     private const uint StatusBadNodeIdInvalid = 0x80330000u;
     private const uint StatusBadInternalError = 0x80020000u;
     private const uint StatusBadCommunicationError = 0x80050000u;
+    private const uint StatusBadNoCommunication = 0x80310000u;
+    private const uint StatusGood = 0x00000000u;
 
     private readonly SemaphoreSlim _gate = new(1, 1);
 
@@ -150,20 +153,82 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     internal SemaphoreSlim Gate => _gate;
 
     /// <summary>
-    ///     Test-only seam to swap the active <see cref="Session"/> the way
-    ///     <see cref="OnReconnectComplete"/> does at runtime, so the stale-session race the
-    ///     read/history paths guard against (re-read Session inside <c>_gate</c>, not before)
-    ///     can be exercised deterministically without standing up a live reconnect handler.
-    ///     Production code never calls this — the SDK reconnect handler owns the real swap.
+    ///     Test-only seam to swap the active <see cref="Session"/> the way a re-initialise does at
+    ///     runtime, so the stale-session race the read/history paths guard against (re-read Session
+    ///     inside <c>_gate</c>, not before) can be exercised deterministically without a live connect.
+    ///     Production code never calls this — <see cref="InitializeAsync"/> owns the real swap.
     /// </summary>
     /// <param name="session">The session instance to install, or <c>null</c> to clear it.</param>
     internal void SetSessionForTest(ISession? session) => Session = session;
 
+    /// <summary>Test-only seam (#670): the most recent in-place recovery task.</summary>
+    internal Task RecoveryTaskForTest => _recoveryTask;
+
+    /// <summary>Test-only seam (#670 review M1): raise the session's PublishError as the SDK publish loop does.</summary>
+    /// <param name="status">The publish error status.</param>
+    /// <param name="subscriptionId">The subscription id the error names (0 = none).</param>
+    internal void RaisePublishErrorForTest(StatusCode status, uint subscriptionId = 0) =>
+        HandlePublishError(Session, status, subscriptionId);
+
+    /// <summary>Test-only seam (#670): install a health snapshot as a completed connect would.</summary>
+    /// <param name="health">The health to install.</param>
+    internal void SetHealthForTest(DriverHealth health) => _health = health;
+
+    /// <summary>Test-only seam (#670): raise the session keep-alive callback as the SDK timer does.</summary>
+    /// <param name="status">The keep-alive status.</param>
+    /// <param name="sender">The raising session; defaults to the current one.</param>
+    internal void RaiseKeepAliveForTest(StatusCode status, ISession? sender = null) =>
+        OnKeepAlive(sender ?? Session!, new KeepAliveEventArgs(new ServiceResult(status), ServerState.Running, DateTime.UtcNow));
+
     /// <summary>
-    ///     Test-only seam to install the <see cref="NamespaceMap"/> that connect/reconnect builds at
+    ///     Test-only seam (#670): register a data subscription over <paramref name="refs"/> as
+    ///     <see cref="SubscribeAsync"/> would, without the wire create.
+    /// </summary>
+    /// <param name="refs">The RawPaths subscribed.</param>
+    /// <returns>The subscription handle.</returns>
+    internal ISubscriptionHandle AddSubscriptionForTest(IReadOnlyList<string> refs)
+    {
+        var id = Interlocked.Increment(ref _nextSubscriptionId);
+        var handle = new OpcUaSubscriptionHandle(id);
+        var subscription = new Subscription(telemetry: null!, new SubscriptionOptions { DisplayName = $"opcua-sub-{id}" });
+        var items = new List<MonitoredItemNotificationHandle>();
+        foreach (var r in refs)
+        {
+            var item = new MonitoredItem(telemetry: null!, new MonitoredItemOptions { DisplayName = r }) { Handle = r };
+            MonitoredItemNotificationEventHandler h = (mi, args) => OnMonitoredItemNotification(handle, mi, args);
+            item.Notification += h;
+            items.Add(new MonitoredItemNotificationHandle(item, h));
+        }
+        _subscriptions[id] = new RemoteSubscription(subscription, handle, items);
+        return handle;
+    }
+
+    /// <summary>Test-only seam (#670): deliver one data notification as the SDK publish thread would.</summary>
+    /// <param name="handle">The subscription handle.</param>
+    /// <param name="fullRef">The RawPath.</param>
+    /// <param name="value">The delivered value.</param>
+    internal void DeliverNotificationForTest(ISubscriptionHandle handle, string fullRef, DataValue value) =>
+        PublishNotification((OpcUaSubscriptionHandle)handle, fullRef, value);
+
+    /// <summary>Test-only seam (#670): register an alarm subscription over <paramref name="refs"/> as
+    /// <see cref="SubscribeAlarmsAsync"/> would, without the wire create.</summary>
+    /// <param name="refs">The alarm RawPaths subscribed.</param>
+    internal void AddAlarmSubscriptionForTest(IReadOnlyList<string> refs)
+    {
+        var id = Interlocked.Increment(ref _nextAlarmSubscriptionId);
+        var handle = new OpcUaAlarmSubscriptionHandle(id);
+        BeginAlarmSubscription(refs);
+        var item = new MonitoredItem(telemetry: null!, new MonitoredItemOptions { DisplayName = "Server/Events" });
+        _alarmSubscriptions[id] = new RemoteAlarmSubscription(
+            new Subscription(telemetry: null!, new SubscriptionOptions { DisplayName = $"{AlarmSubscriptionNamePrefix}{id}" }),
+            handle, item, (_, _) => { });
+    }
+
+    /// <summary>
+    ///     Test-only seam to install the <see cref="NamespaceMap"/> that connect builds at
     ///     runtime, so the v3 RawPath → upstream node id resolution can be exercised without a live
     ///     connect (which is what populates <see cref="_namespaceMap"/>). Production code never calls
-    ///     this — <see cref="InitializeAsync"/> / <see cref="OnReconnectComplete"/> own the real build.
+    ///     this — <see cref="InitializeAsync"/> owns the real build.
     /// </summary>
     /// <param name="map">The namespace map to install, or <c>null</c> to clear it.</param>
     internal void SetNamespaceMapForTest(NamespaceMap? map) => _namespaceMap = map;
@@ -191,13 +256,65 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     /// <summary>The <see cref="CertificateValidator"/> that owns <see cref="_certValidationHandler"/>.</summary>
     private CertificateValidator? _certValidatorRef;
     /// <summary>
-    ///     SDK-provided reconnect handler that owns the retry loop + session-transfer machinery
-    ///     when the session's keep-alive channel reports a bad status. Null outside the
-    ///     reconnecting window; constructed lazily inside the keep-alive handler. Guarded by
-    ///     <see cref="_probeLock"/> — keep-alive callbacks fire from the SDK timer thread and
-    ///     can race a check-then-set if left unsynchronized.
+    ///     Gitea #670 — 1 while the CURRENT session's keep-alive is bad (an outage is in progress), else 0. The
+    ///     0→1 edge publishes the outage once (Bad per subscribed ref + a quality-only alarm annotation); the 1→0
+    ///     edge on a good keep-alive recovers in place. Reset whenever the session is released, so a fresh session
+    ///     reports its own loss. There is deliberately no in-driver reconnect any more: a session that never comes
+    ///     back keeps health Degraded with <see cref="DriverHealth.LastSuccessfulRead"/> frozen, which is exactly what
+    ///     the host's #527 comms-loss watchdog trips on (→ ShutdownAsync + InitializeAsync + re-subscribe).
     /// </summary>
-    private SessionReconnectHandler? _reconnectHandler;
+    private int _sessionLost;
+
+    /// <summary>
+    ///     #670 review M1 — 1 once a tracked subscription is known dead while the SESSION lives (it stopped
+    ///     publishing past its keep-alive window, or the server said BadNoSubscription / BadSubscriptionIdInvalid —
+    ///     e.g. an outage longer than the subscription lifetime but shorter than the session timeout). Sticky until
+    ///     the session is released: health stays Degraded and NOTHING advances <see cref="DriverHealth.LastSuccessfulRead"/>
+    ///     (good keep-alives would otherwise keep a driver whose data plane is dead looking alive forever), so the
+    ///     host's #527 watchdog re-initialises the driver and the re-subscribe recreates the subscription.
+    /// </summary>
+    private int _subscriptionLost;
+
+    /// <summary>#670 review M1 — ticks of the first good keep-alive that saw a tracked subscription not publishing
+    /// (0 = none stalled). Only the keep-alive thread and the session release touch it.</summary>
+    private long _publishingStoppedSinceTicks;
+
+    /// <summary>The last in-place recovery started (#670). Test seam: lets a test await it deterministically.</summary>
+    private Task _recoveryTask = Task.CompletedTask;
+
+    /// <summary>Test seam (#670 review M1): whether a subscription has stopped publishing. Production asks the SDK.</summary>
+    internal Func<Subscription, bool> IsPublishingStopped { get; set; } = static s => s.Created && s.PublishingStopped;
+
+    /// <summary>Test seam (#670 review M1): how long a subscription may keep reporting stopped (measured on the
+    /// driver's keep-alive cadence) before it is declared dead. Production: one more subscription keep-alive window
+    /// plus 5 s, on top of the window the SDK's own <c>PublishingStopped</c> already waits.</summary>
+    internal Func<Subscription, TimeSpan> PublishingStallGrace { get; set; } = static s =>
+        TimeSpan.FromMilliseconds(Math.Max(s.CurrentPublishingInterval, 1) * (s.CurrentKeepAliveCount + 1d))
+        + TimeSpan.FromSeconds(5);
+
+    /// <summary>#670 — the last value delivered per subscribed RawPath, so an outage can publish
+    /// <c>BadNoCommunication</c> with the value retained (the same shape the host's own comms-loss fan uses).</summary>
+    private readonly System.Collections.Concurrent.ConcurrentDictionary<string, DataValueSnapshot> _lastValues =
+        new(StringComparer.Ordinal);
+
+    /// <summary>#670 — refs published Bad by the current outage and not yet re-published Good. A notification
+    /// clears its ref (fresh data beats the recovery read); the recovery read publishes only what is left.</summary>
+    private readonly System.Collections.Concurrent.ConcurrentDictionary<string, byte> _awaitingRecovery =
+        new(StringComparer.Ordinal);
+
+    /// <summary>Ticks of the last <see cref="DriverHealth.LastSuccessfulRead"/> advance from a notification —
+    /// throttles the per-notification health write to at most one per second.</summary>
+    private long _lastEvidenceStampTicks;
+
+    /// <summary>Clock for liveness stamps. Test seam (#670); production uses <see cref="TimeProvider.System"/>.</summary>
+    internal TimeProvider Clock { get; set; } = TimeProvider.System;
+
+    /// <summary>
+    ///     SDK telemetry for this instance's sessions: SDK warnings (keep-alive / publish / channel errors) go to the
+    ///     driver's own logger (#670 — the shared <see cref="SdkTelemetry.Context"/> has no provider, which made every
+    ///     SDK-side session error invisible).
+    /// </summary>
+    private readonly ITelemetryContext _sdkTelemetry;
 
     /// <summary>
     ///     Bidirectional namespace map built at connect time from <c>session.NamespaceUris</c>.
@@ -314,12 +431,11 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
                     "Tried:\n  " + string.Join("\n  ", attemptErrors),
                     attemptErrors.Select(e => new InvalidOperationException(e)));
 
-            // Wire the session's keep-alive channel into HostState + the reconnect trigger.
-            // OPC UA keep-alives are authoritative for session liveness: the SDK pings on
-            // KeepAliveInterval and sets KeepAliveStopped when N intervals elapse without a
-            // response. On a bad keep-alive the driver spins up a SessionReconnectHandler
-            // which transparently retries + swaps the underlying session. Subscriptions move
-            // via TransferSubscriptions so local MonitoredItem handles stay valid.
+            // Wire the session's keep-alive channel into HostState + health (#670). OPC UA keep-alives
+            // are authoritative for session liveness: the SDK pings on KeepAliveInterval. A good one is
+            // liveness evidence (advances LastSuccessfulRead); a bad one degrades health and publishes the
+            // outage. Recovery from a session that does not come back is the host's comms-loss watchdog
+            // (ShutdownAsync + InitializeAsync + re-subscribe) — there is no in-driver reconnect handler.
             _keepAliveHandler = OnKeepAlive;
             session.KeepAlive += _keepAliveHandler;
             AttachDiagnostics(session);
@@ -355,9 +471,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     /// </summary>
     private async Task<ApplicationConfiguration> BuildApplicationConfigurationAsync(CancellationToken ct)
     {
-        // The ITelemetryContext overload is the non-obsolete ctor; an empty builder keeps SDK
-        // internals off any logging provider (mirrors Client.Shared's ClientTelemetry).
-        var app = new ApplicationInstance(SdkTelemetry.Context)
+        // The ITelemetryContext overload is the non-obsolete ctor; SDK warnings reach the driver's
+        // logger (#670) when one was supplied.
+        var app = new ApplicationInstance(_sdkTelemetry)
         {
             ApplicationName = _options.SessionName,
             ApplicationType = ApplicationType.Client,
@@ -681,7 +797,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         endpointConfig.OperationTimeout = (int)_options.Timeout.TotalMilliseconds;
         var endpoint = new ConfiguredEndpoint(null, selected, endpointConfig);
 
-        var session = await new DefaultSessionFactory(SdkTelemetry.Context).CreateAsync(
+        var session = await new DefaultSessionFactory(_sdkTelemetry).CreateAsync(
             appConfig,
             endpoint,
             false,                                              // updateBeforeConnect
@@ -795,30 +911,15 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     }
 
     /// <summary>
-    ///     Cancels any in-flight reconnect, detaches the keep-alive handler and closes + disposes the current
-    ///     session. The session part of <see cref="ShutdownAsync"/>; also run at the top of
-    ///     <see cref="InitializeAsync"/> so a re-initialise on a live driver (#630) does not orphan the
-    ///     previous session and its reconnect handler. Subscriptions are NOT touched here.
+    ///     Detaches the keep-alive handler and closes + disposes the current session. The session part of
+    ///     <see cref="ShutdownAsync"/>; also run at the top of <see cref="InitializeAsync"/> so a re-initialise on a
+    ///     live driver (#630) does not orphan the previous session. Subscriptions are NOT touched here. Clears the
+    ///     #670 outage latch, so the next session reports its own loss.
     /// </summary>
     private async Task ReleaseSessionAsync(CancellationToken cancellationToken)
     {
-        // Abort any in-flight reconnect attempts before touching the session — BeginReconnect's
-        // retry loop holds a reference to the current session and would fight Session.CloseAsync
-        // if left spinning. Take the handler under _probeLock so a keep-alive callback racing
-        // through OnKeepAlive can't arm a fresh handler after we've torn this one down.
-        SessionReconnectHandler? handlerToCancel;
-        lock (_probeLock)
-        {
-            handlerToCancel = _reconnectHandler;
-            _reconnectHandler = null;
-        }
-        try { handlerToCancel?.CancelReconnect(); } catch { }
-        handlerToCancel?.Dispose();
-
-        // Take the session reference under _probeLock before touching it, so we can't race
-        // an OnReconnectComplete that is simultaneously swapping to a new session. We clear
-        // Session to null here so any concurrent caller that checks inside _gate sees null
-        // immediately after shutdown begins.
+        // Take the session reference under _probeLock and clear Session to null so any concurrent caller that
+        // checks inside _gate sees null immediately after release begins.
         ISession? sessionToClose;
         lock (_probeLock)
         {
@@ -832,11 +933,29 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
             Session = null;
         }
 
-        try { if (sessionToClose is Session s) await s.CloseAsync(cancellationToken).ConfigureAwait(false); }
+        // #670: bound the close — against a dead upstream (the comms-loss watchdog's whole use case) an unbounded
+        // CloseSession would sit out the full operation timeout on every reconnect attempt — and skip it outright
+        // when the keep-alive already says the session is unreachable (review L3).
+        var sessionKnownLost = Volatile.Read(ref _sessionLost) == 1;
+        try
+        {
+            if (sessionToClose is Session s && !sessionKnownLost)
+            {
+                using var closeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+                closeCts.CancelAfter(SessionCloseTimeout);
+                await s.CloseAsync(closeCts.Token).ConfigureAwait(false);
+            }
+        }
         catch { /* best-effort */ }
         try { sessionToClose?.Dispose(); } catch { }
+        Interlocked.Exchange(ref _sessionLost, 0);
+        Interlocked.Exchange(ref _subscriptionLost, 0);
+        Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
     }
 
+    /// <summary>Upper bound on the best-effort wire calls a teardown makes against a possibly-dead upstream.</summary>
+    private static readonly TimeSpan SessionCloseTimeout = TimeSpan.FromSeconds(5);
+
     /// <inheritdoc />
     public async Task ShutdownAsync(CancellationToken cancellationToken)
     {
@@ -846,11 +965,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         // after close anyway. Before deleting each subscription we detach the Notification
         // handlers we attached at subscribe time so the SDK's invocation list no longer
         // holds the driver instance through the closure.
+        // #670: against a session already known lost (bad keep-alive) the wire deletes cannot succeed and would each
+        // sit out the operation timeout — skip them; disposing the session releases the local objects.
+        var sessionAlive = Volatile.Read(ref _sessionLost) == 0;
         foreach (var rs in _subscriptions.Values)
         {
             DetachNotificationHandlers(rs.ItemHandlers);
-            try { await rs.Subscription.DeleteAsync(silent: true, cancellationToken).ConfigureAwait(false); }
-            catch { /* best-effort */ }
+            if (sessionAlive) await DeleteSubscriptionBestEffortAsync(rs.Subscription, cancellationToken).ConfigureAwait(false);
         }
         _subscriptions.Clear();
 
@@ -858,10 +979,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         {
             try { ras.EventItem.Notification -= ras.Handler; }
             catch { /* best-effort */ }
-            try { await ras.Subscription.DeleteAsync(silent: true, cancellationToken).ConfigureAwait(false); }
-            catch { /* best-effort */ }
+            if (sessionAlive) await DeleteSubscriptionBestEffortAsync(ras.Subscription, cancellationToken).ConfigureAwait(false);
         }
         _alarmSubscriptions.Clear();
+        // The host re-subscribes after a reinit and the new subscriptions deliver initial values; nothing of the
+        // previous connection's value bookkeeping applies to them.
+        _lastValues.Clear();
+        _awaitingRecovery.Clear();
         // #619 re-review: the last-told alarm states (and ack targets) deliberately SURVIVE a shutdown. A
         // ReinitializeAsync (comms-loss watchdog / Force Reconnect) is followed by the host re-subscribing the
         // same set over condition nodes it did NOT rebuild; forgetting here would make the post-subscribe
@@ -885,6 +1009,18 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
     }
 
+    /// <summary>Best-effort, bounded delete of one subscription (shutdown path).</summary>
+    private static async Task DeleteSubscriptionBestEffortAsync(Subscription subscription, CancellationToken cancellationToken)
+    {
+        try
+        {
+            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            cts.CancelAfter(SessionCloseTimeout);
+            await subscription.DeleteAsync(silent: true, cts.Token).ConfigureAwait(false);
+        }
+        catch { /* best-effort */ }
+    }
+
     /// <inheritdoc />
     public DriverHealth GetHealth() => _health;
 
@@ -899,22 +1035,35 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         // call to this reference — a reconnect can swap Session while we wait on _gate. The
         // session actually used is re-read inside the gate.
         _ = RequireSession();
+        var (results, _) = await ReadCoreAsync(fullReferences, cancellationToken).ConfigureAwait(false);
+        return results;
+    }
+
+    /// <summary>
+    ///     The read itself, also used by the #670 in-place recovery. <c>TransportOk</c> is false when the wire read
+    ///     did not happen or threw (every result is then a local <c>BadCommunicationError</c>, not upstream data).
+    ///     <paramref name="expectedSession"/>, when given, must still be the current session inside the gate — a
+    ///     recovery overtaken by a re-initialise neither reads on, nor writes the health of, the new session.
+    /// </summary>
+    private async Task<(DataValueSnapshot[] Results, bool TransportOk)> ReadCoreAsync(
+        IReadOnlyList<string> fullReferences, CancellationToken cancellationToken, ISession? expectedSession = null)
+    {
         var results = new DataValueSnapshot[fullReferences.Count];
-        var now = DateTime.UtcNow;
+        var now = Clock.GetUtcNow().UtcDateTime;
 
         await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
         try
         {
             // Re-read Session inside the critical section: if a reconnect completed while we
-            // were blocked on _gate, OnReconnectComplete has already swapped in the new
+            // were blocked on _gate, a re-initialise has already swapped in the new
             // session. NodeId parsing is namespace-relative, so it must also use the current
             // session's namespace table.
             var session = Session;
-            if (session is null)
+            if (session is null || (expectedSession is not null && !ReferenceEquals(session, expectedSession)))
             {
                 for (var i = 0; i < fullReferences.Count; i++)
                     results[i] = new DataValueSnapshot(null, StatusBadCommunicationError, null, now);
-                return results;
+                return (results, false);
             }
 
             // Resolve each RawPath reference (v3) to a live NodeId against the current session.
@@ -934,7 +1083,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
                 indexMap.Add(i);
             }
 
-            if (toSend.Count == 0) return results;
+            if (toSend.Count == 0) return (results, true);
 
             try
             {
@@ -973,10 +1122,11 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
                     results[r] = new DataValueSnapshot(null, StatusBadCommunicationError, null, now);
                 }
                 _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, ex.Message);
+                return (results, false);
             }
         }
         finally { _gate.Release(); }
-        return results;
+        return (results, true);
     }
 
     // ---- IWritable ----
@@ -1473,7 +1623,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         try
         {
             // Re-read Session inside the critical section: a reconnect completing while we were
-            // blocked on _gate has already swapped in the new session via OnReconnectComplete.
+            // blocked on _gate has already swapped in the new session via a re-initialise.
             var session = Session
                 ?? throw new InvalidOperationException(
                     "OpcUaClient session was lost before the subscription could be created.");
@@ -1557,17 +1707,60 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         if (args.NotificationValue is not MonitoredItemNotification mn) return;
         var dv = mn.Value;
         if (dv is null) return;
-        var fullRef = (item.Handle as string) ?? item.DisplayName ?? string.Empty;
+        PublishNotification(handle, RefOf(item), dv);
+    }
+
+    private static string RefOf(MonitoredItem item) => (item.Handle as string) ?? item.DisplayName ?? string.Empty;
+
+    private void PublishNotification(OpcUaSubscriptionHandle handle, string fullRef, DataValue dv)
+    {
         _diag.Record(OpcUaClientDiagnostics.DataChange);
         if (StatusCode.IsBad(dv.StatusCode)) _diag.Record(OpcUaClientDiagnostics.BadStatusValue);
+        // #670: a delivered notification is liveness evidence — the subscription path never calls ReadAsync, so
+        // without this a busy subscription-only driver's LastSuccessfulRead stayed at its connect time.
+        RecordLivenessEvidence();
         var snapshot = new DataValueSnapshot(
             Value: dv.Value,
             StatusCode: dv.StatusCode.Code,
             SourceTimestampUtc: dv.SourceTimestamp == DateTime.MinValue ? null : dv.SourceTimestamp,
             ServerTimestampUtc: dv.ServerTimestamp == DateTime.MinValue ? DateTime.UtcNow : dv.ServerTimestamp);
+        _lastValues[fullRef] = snapshot;
+        _awaitingRecovery.TryRemove(fullRef, out _);
         OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullRef, snapshot));
     }
 
+    /// <summary>
+    ///     #670 — advance <see cref="DriverHealth.LastSuccessfulRead"/> on upstream traffic (a notification), at most
+    ///     once per second. The state is left as it is: a notification is evidence of comms, but the keep-alive
+    ///     is what declares an outage over.
+    /// </summary>
+    private void RecordLivenessEvidence()
+    {
+        // A dead subscription (M1) must keep the stamp frozen even while OTHER subscriptions still deliver.
+        if (Volatile.Read(ref _subscriptionLost) == 1) return;
+        var now = Clock.GetUtcNow().UtcDateTime;
+        var last = Interlocked.Read(ref _lastEvidenceStampTicks);
+        if (now.Ticks - last < TimeSpan.TicksPerSecond && last != 0) return;
+        Interlocked.Exchange(ref _lastEvidenceStampTicks, now.Ticks);
+        UpdateHealth(h => h.State is DriverState.Healthy or DriverState.Degraded ? h with { LastSuccessfulRead = now } : null);
+    }
+
+    /// <summary>
+    ///     Read-modify-write of <see cref="_health"/> as a CAS loop (#670 review L1): the keep-alive timer, publish
+    ///     threads and the recovery task all derive the next health from the current one.
+    /// </summary>
+    /// <param name="next">Maps the current health to the next one, or null to leave it unchanged.</param>
+    private void UpdateHealth(Func<DriverHealth, DriverHealth?> next)
+    {
+        while (true)
+        {
+            var current = Volatile.Read(ref _health);
+            var proposed = next(current);
+            if (proposed is null) return;
+            if (ReferenceEquals(Interlocked.CompareExchange(ref _health, proposed, current), current)) return;
+        }
+    }
+
     /// <summary>
     ///     Live data-change subscription bookkeeping. Holds the SDK <see cref="Subscription"/>,
     ///     the local handle, and the per-MonitoredItem (item, handler) pairs so
@@ -1628,8 +1821,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         // match in O(1) without re-parsing on every event.
         // #619: under v3 the host passes RawPaths, never upstream node ids. Translate each to the upstream
         // source id its tag authored (TagConfig.nodeId) so the per-event match compares like with like.
-        // Keys are server-stable (AlarmIndexKey) so a namespace reorder across an in-place SDK reconnect —
-        // which transfers this subscription without re-running SubscribeAlarmsAsync — cannot orphan them.
+        // Keys are server-stable (AlarmIndexKey) so a remote namespace reorder cannot orphan them.
         var sessionForResolve = Session;
         var mapForResolve = _namespaceMap;
         var sourceFilter = BuildAlarmSourceIndex(sourceNodeIds, raw =>
@@ -1709,7 +1901,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         try
         {
             // Re-read Session inside the critical section: a reconnect completing while we were
-            // blocked on _gate has already swapped in the new session via OnReconnectComplete.
+            // blocked on _gate has already swapped in the new session via a re-initialise.
             var session = Session
                 ?? throw new InvalidOperationException(
                     "OpcUaClient session was lost before the alarm subscription could be created.");
@@ -1815,7 +2007,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         try
         {
             // Re-read Session inside the critical section: a reconnect completing while we were
-            // blocked on _gate has already swapped in the new session via OnReconnectComplete.
+            // blocked on _gate has already swapped in the new session via a re-initialise.
             // ConditionId parsing is namespace-relative, so it must use the current session's
             // namespace table too.
             var session = Session;
@@ -1914,9 +2106,10 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     {
         if (args.NotificationValue is not EventFieldList efl) return;
         // #662 review: only a subscription SubscribeAlarmsAsync completed (and nothing has since released) may
-        // deliver. A half-built one whose cleanup failed against a broken server can survive on the session and
-        // be transferred by an SDK reconnect; without this it would feed RouteAlarmEvent alongside the real one.
+        // deliver. A half-built one whose cleanup failed against a broken server can survive on the session;
+        // without this it would feed RouteAlarmEvent alongside the real one.
         if (!_alarmSubscriptions.ContainsKey(handle.Id)) return;
+        RecordLivenessEvidence();
         RouteAlarmEvent(efl.EventFields, sourceFilter, handle);
     }
 
@@ -2063,10 +2256,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     }
 
     /// <summary>
-    ///     #619 — ConditionRefresh every alarm subscription on the current session: after an in-place reconnect
-    ///     (transferred or SDK-recreated subscriptions — matched by display name, since a recreate clones the
-    ///     <see cref="Subscription"/> objects) and on a RefreshRequired event. Events missed during the outage
-    ///     reach the host through the normal deduplicated path.
+    ///     #619 — ConditionRefresh every alarm subscription on the current session (matched by display name): after
+    ///     a keep-alive outage the session survived (#670) and on a RefreshRequired event. Events missed during the
+    ///     outage reach the host through the normal deduplicated path.
     /// </summary>
     /// <param name="cancellationToken">Cancellation.</param>
     /// <returns>A task completing when every refresh call has returned.</returns>
@@ -2269,7 +2461,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     internal readonly record struct AlarmAckTarget(string ConditionRef, byte[] EventId);
 
     /// <summary>#619 — the key both sides of the alarm index use: server-stable (<c>nsu=</c>) when a map is
-    /// available, so a remote namespace-table reorder across an in-place reconnect cannot orphan it.</summary>
+    /// available, so a remote namespace-table reorder cannot orphan it.</summary>
     /// <param name="map">The current session's namespace map, or null.</param>
     /// <param name="nodeId">The NodeId to key.</param>
     /// <returns>The index key.</returns>
@@ -2341,7 +2533,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
                 foreach (var nd in data)
                     _diag.Record(OpcUaClientDiagnostics.Notification, CountNotifications(nd));
         };
-        _publishErrorHandler ??= (_, _) => _diag.Record(OpcUaClientDiagnostics.PublishError);
+        _publishErrorHandler ??= OnPublishError;
         session.Notification -= _notificationHandler;
         session.Notification += _notificationHandler;
         session.PublishError -= _publishErrorHandler;
@@ -2367,174 +2559,246 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     };
 
     /// <summary>
-    ///     Session keep-alive handler. On a healthy ping, bumps HostState back to Running
-    ///     (typical bounce after a transient network blip). On a bad ping, starts the SDK's
-    ///     <see cref="SessionReconnectHandler"/> which retries on the configured period +
-    ///     fires <see cref="OnReconnectComplete"/> when it lands a new session.
+    ///     Session keep-alive handler (Gitea #670). A good keep-alive is liveness evidence: it advances
+    ///     <see cref="DriverHealth.LastSuccessfulRead"/>, and after a bad one it ends the outage in place (the session
+    ///     survived — e.g. a network blip): health back to Healthy and the outage's Bad refs re-read and re-published
+    ///     Good, because a server whose subscription survived will not resend values that did not change. A bad
+    ///     keep-alive degrades health (stamp preserved) and, once per outage, publishes <c>BadNoCommunication</c> for
+    ///     every subscribed ref (value retained) plus a quality-only annotation on every subscribed alarm condition.
     /// </summary>
+    /// <remarks>
+    ///     There is deliberately no <c>SessionReconnectHandler</c> any more. In SDK 1.5.378 it retried forever, its
+    ///     session-recreate path cloned subscriptions the driver never re-tracked, and it kept health Healthy while
+    ///     it did so — the #670 symptom. A session that does not come back keeps health Degraded with the read stamp
+    ///     frozen, which is exactly what the host's #527 comms-loss watchdog trips on: ShutdownAsync + InitializeAsync
+    ///     + re-subscribe, whose initial values come back Good.
+    /// </remarks>
     private void OnKeepAlive(ISession sender, KeepAliveEventArgs e)
     {
+        // A late callback from a session already released (re-initialise) says nothing about the current one.
+        if (!ReferenceEquals(sender, Session)) return;
+        var now = Clock.GetUtcNow().UtcDateTime;
+
         if (!ServiceResult.IsBad(e.Status))
         {
             TransitionTo(HostState.Running);
+            // M1: the session answers but a subscription is dead — no stamp, no recovery; the watchdog reinit is the fix.
+            if (Volatile.Read(ref _subscriptionLost) == 1)
+            {
+                Interlocked.Exchange(ref _sessionLost, 0);
+                return;
+            }
+
+            if (Interlocked.Exchange(ref _sessionLost, 0) == 1)
+            {
+                UpdateHealth(_ => new DriverHealth(DriverState.Healthy, now, null));
+                // A fresh grace: every subscription reports stopped right after an outage until its next message.
+                Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
+                _diag.Record(OpcUaClientDiagnostics.KeepAliveRecovered);
+                _logger.LogInformation(
+                    "OpcUaClientDriver '{DriverInstanceId}': upstream keep-alive recovered on the same session; " +
+                    "re-reading the refs published Bad during the outage.", _driverInstanceId);
+                _recoveryTask = Task.Run(() => RecoverAfterOutageAsync(sender));
+                return;
+            }
+
+            if (FindStalledSubscription(now) is { } stalled)
+            {
+                DeclareSubscriptionLost($"subscription '{stalled}' stopped publishing while the session stayed up", now);
+                return;
+            }
+
+            UpdateHealth(h => h.State is DriverState.Healthy or DriverState.Degraded ? h with { LastSuccessfulRead = now } : null);
             return;
         }
 
+        Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
         TransitionTo(HostState.Stopped);
         _diag.Record(OpcUaClientDiagnostics.KeepAliveMiss);
+        UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"keep-alive: {e.Status}"));
+        if (Interlocked.Exchange(ref _sessionLost, 1) == 1) return;
+        if (Volatile.Read(ref _subscriptionLost) == 1) return; // already published Bad for this connection
 
-        // Kick off the SDK's reconnect loop exactly once per drop. Keep-alive callbacks fire
-        // from the SDK keep-alive timer thread and the SDK can fire this handler repeatedly
-        // while the channel stays down — the check-then-set must be atomic, otherwise two
-        // callbacks both observe null, both construct a SessionReconnectHandler, and the
-        // second assignment leaks the first (its retry loop keeps running, unreferenced and
-        // never disposed). Guard with _probeLock.
-        SessionReconnectHandler handler;
-        lock (_probeLock)
-        {
-            if (_reconnectHandler is not null || _disposed) return;
-            handler = new SessionReconnectHandler(telemetry: null!,
-                reconnectAbort: false,
-                maxReconnectPeriod: (int)TimeSpan.FromMinutes(2).TotalMilliseconds);
-            _reconnectHandler = handler;
-        }
-
-        // BeginReconnect is started outside the lock — it does no _reconnectHandler mutation
-        // and we don't want to hold _probeLock across an SDK call.
-        _diag.Record(OpcUaClientDiagnostics.ReconnectAttempt);
-        handler.BeginReconnect(
-            sender,
-            (int)_options.ReconnectPeriod.TotalMilliseconds,
-            OnReconnectComplete);
+        _logger.LogWarning(
+            "OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
+            "are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
+            _driverInstanceId, e.Status);
+        PublishCommsLoss(now);
     }
 
     /// <summary>
-    ///     Called by <see cref="SessionReconnectHandler"/> when its retry loop has either
-    ///     successfully swapped to a new session or given up. Reads the new session off
-    ///     <c>handler.Session</c>, unwires the old keep-alive hook, rewires for the new
-    ///     one, and tears down the handler. Subscription migration is already handled
-    ///     inside the SDK via <c>TransferSubscriptions</c> (the SDK calls it automatically
-    ///     when <see cref="Session.TransferSubscriptionsOnReconnect"/> is <c>true</c>,
-    ///     which is the default).
+    ///     #670 review M1 — the display name of a tracked subscription that has reported "publishing stopped" for
+    ///     longer than its grace, measured across consecutive good keep-alives; null when none has.
     /// </summary>
-    private void OnReconnectComplete(object? sender, EventArgs e)
+    private string? FindStalledSubscription(DateTime now)
     {
-        if (sender is not SessionReconnectHandler handler) return;
-        var newSession = handler.Session;
-
-        // All mutations to Session and _reconnectHandler run under _probeLock so
-        // OnReconnectComplete, OnKeepAlive, and ShutdownAsync cannot race each other:
-        // a session swap visible to concurrent ReadAsync/WriteAsync/DiscoverAsync callers
-        // (which re-read Session inside _gate) must be atomic w.r.t. disposal and
-        // re-arming.
-        ISession? oldSession;
-        lock (_probeLock)
+        Subscription? worst = null;
+        var grace = TimeSpan.Zero;
+        foreach (var sub in _subscriptions.Values.Select(r => r.Subscription)
+                     .Concat(_alarmSubscriptions.Values.Select(r => r.Subscription)))
         {
-            oldSession = Session;
-
-            // Rewire keep-alive before swapping the reference so a hot keep-alive can't
-            // fire against the old session after we've already assigned the new one.
-            if (oldSession is not null && _keepAliveHandler is not null)
-            {
-                try { oldSession.KeepAlive -= _keepAliveHandler; } catch { }
-            }
-            if (newSession is not null && _keepAliveHandler is not null)
-            {
-                newSession.KeepAlive += _keepAliveHandler;
-            }
-            // Detach-then-attach: the SDK may hand back the SAME session object, and -= on a handler that
-            // is not attached is a no-op, so this never double-counts.
-            if (oldSession is not null) DetachDiagnostics(oldSession);
-            if (newSession is not null) AttachDiagnostics(newSession);
-
-            Session = newSession;
-
-            // Retire the handler that just finished.
-            if (ReferenceEquals(_reconnectHandler, handler))
-            {
-                _reconnectHandler.Dispose();
-                _reconnectHandler = null;
-            }
+            if (!IsPublishingStopped(sub)) continue;
+            var g = PublishingStallGrace(sub);
+            if (worst is null || g > grace) { worst = sub; grace = g; }
         }
 
-        if (newSession is not null)
+        if (worst is null)
         {
-            // Reconnect succeeded. Rebuild the namespace map from the *new* session: the
-            // remote server may have reordered its namespace table across the restart that
-            // caused the drop (driver-specs.md §8). Stable nsu= references stored in the
-            // address space re-resolve correctly against this fresh map. v3: re-ingest the
-            // authored RawTags so RawPath resolution survives the reconnect too.
-            _namespaceMap = NamespaceMap.FromSession(newSession, _options.RawTags);
-            RecordSkippedTags(_namespaceMap);
-            TransitionTo(HostState.Running);
-            _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
-            _diag.Record(OpcUaClientDiagnostics.ReconnectSuccess);
-            // #619: transitions during the outage were missed (a recreated subscription starts empty; a transferred
-            // one may have overflowed). ConditionRefresh re-reports current states; unchanged ones dedup to nothing.
-            if (!_alarmSubscriptions.IsEmpty)
-                _ = Task.Run(() => RefreshAllAlarmSubscriptionsAsync(CancellationToken.None));
+            Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
+            return null;
+        }
+
+        var since = Interlocked.CompareExchange(ref _publishingStoppedSinceTicks, now.Ticks, 0);
+        if (since == 0) since = now.Ticks;
+        return now.Ticks - since >= grace.Ticks ? worst.DisplayName ?? $"#{worst.Id}" : null;
+    }
+
+    /// <summary>
+    ///     #670 review M1 — the server dropped a subscription the driver still tracks. Degraded with the read stamp
+    ///     frozen (sticky until the session is released), and the outage published once, so the host's watchdog
+    ///     re-initialises and re-subscribes.
+    /// </summary>
+    private void DeclareSubscriptionLost(string reason, DateTime now)
+    {
+        if (Interlocked.Exchange(ref _subscriptionLost, 1) == 1) return;
+        UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"subscription lost: {reason}"));
+        _logger.LogWarning(
+            "OpcUaClientDriver '{DriverInstanceId}': {Reason}; subscribed values are BadNoCommunication until the " +
+            "host's comms-loss watchdog re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
+        if (Volatile.Read(ref _sessionLost) == 0) PublishCommsLoss(now);
+    }
+
+    /// <summary>
+    ///     Session PublishError: always counted (#276); a BadNoSubscription / BadSubscriptionIdInvalid naming a
+    ///     subscription the driver tracks means the server dropped it while the session lived (#670 review M1).
+    /// </summary>
+    private void OnPublishError(object? sender, PublishErrorEventArgs e) =>
+        HandlePublishError(sender, e.Status?.StatusCode ?? StatusCodes.Good, e.SubscriptionId);
+
+    private void HandlePublishError(object? sender, StatusCode code, uint subscriptionId)
+    {
+        _diag.Record(OpcUaClientDiagnostics.PublishError);
+        if (sender is not null && !ReferenceEquals(sender, Session)) return;
+        bool tracked;
+        if (code == StatusCodes.BadNoSubscription)
+            tracked = !_subscriptions.IsEmpty || !_alarmSubscriptions.IsEmpty;
+        else if (code == StatusCodes.BadSubscriptionIdInvalid)
+            tracked = _subscriptions.Values.Any(r => r.Subscription.Id == subscriptionId)
+                      || _alarmSubscriptions.Values.Any(r => r.Subscription.Id == subscriptionId);
+        else
             return;
-        }
-
-        // The reconnect handler gave up — its retry loop exhausted the 2-minute
-        // maxReconnectPeriod and invoked the callback with a null Session. Without an
-        // explicit Faulted signal the driver is permanently wedged: no session, no live
-        // keep-alive to re-trigger OnKeepAlive, and the Core never learns it must offer an
-        // operator reinitialize. Surface Faulted so the Core fans out Bad quality and
-        // ReinitializeAsync becomes available, and arm a fresh reconnect attempt against
-        // the last-known session for an always-on gateway rather than abandoning recovery
-        // entirely.
-        TransitionTo(HostState.Faulted);
-        _diag.Record(OpcUaClientDiagnostics.ReconnectExhausted);
-        _health = new DriverHealth(
-            DriverState.Faulted, _health.LastSuccessfulRead,
-            "OPC UA session reconnect exhausted its retry window without recovering. " +
-            "The remote server is unreachable; reinitialize the driver once it is back.");
-
-        if (oldSession is not null && !_disposed)
-            TryRearmReconnect(handler, oldSession);
+        if (tracked)
+            DeclareSubscriptionLost($"server reported {code} for a tracked subscription", Clock.GetUtcNow().UtcDateTime);
     }
 
     /// <summary>
-    ///     Arm a fresh reconnect attempt after a previous handler gave up. The OPC UA Client
-    ///     driver gateways an always-on remote server, so abandoning recovery permanently is
-    ///     the wrong default — a new <see cref="SessionReconnectHandler"/> keeps retrying so
-    ///     the driver self-heals when the remote returns, while the Faulted health set by the
-    ///     caller still lets an operator force a clean reinitialize in the meantime.
+    ///     #670 — publish the start of an outage: <c>BadNoCommunication</c> once per (subscription, ref) with the last
+    ///     delivered value retained, and a quality-only annotation on every subscribed alarm condition (no state
+    ///     change — an outage says nothing about whether the alarm is active).
     /// </summary>
-    private void TryRearmReconnect(SessionReconnectHandler exhausted, ISession lastSession)
+    private void PublishCommsLoss(DateTime now)
     {
-        SessionReconnectHandler handler;
-        lock (_probeLock)
+        foreach (var rs in _subscriptions.Values)
         {
-            // Only re-arm if no other handler took over and we aren't shutting down.
-            if (_disposed || (_reconnectHandler is not null && !ReferenceEquals(_reconnectHandler, exhausted)))
-                return;
-            handler = new SessionReconnectHandler(telemetry: null!,
-                reconnectAbort: false,
-                maxReconnectPeriod: (int)TimeSpan.FromMinutes(2).TotalMilliseconds);
-            _reconnectHandler = handler;
+            foreach (var pair in rs.ItemHandlers)
+            {
+                var fullRef = RefOf(pair.Item);
+                _lastValues.TryGetValue(fullRef, out var last);
+                _awaitingRecovery[fullRef] = 0;
+                RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, new DataValueSnapshot(
+                    last?.Value, StatusBadNoCommunication, last?.SourceTimestampUtc, now)));
+            }
         }
+        AnnotateAlarmQuality(StatusBadNoCommunication, now);
+    }
 
+    /// <summary>
+    ///     #670 — the session survived the outage: re-read the refs still awaiting recovery and publish them (a ref
+    ///     a notification already refreshed is skipped — fresh data beats the read), restore alarm-condition quality
+    ///     and ConditionRefresh so transitions missed during the outage are re-reported (deduplicated). A failed
+    ///     read re-arms the outage so the next good keep-alive tries again.
+    /// </summary>
+    /// <param name="session">The session whose keep-alive recovered. Every effect — publishing, re-arming the
+    /// latch — is conditional on it still being the current session and still out of outage (#670 review M2/L2): a
+    /// recovery overtaken by a re-initialise must not leak its outcome onto the NEW session.</param>
+    private async Task RecoverAfterOutageAsync(ISession session)
+    {
+        bool Current() => ReferenceEquals(session, Session) && Volatile.Read(ref _sessionLost) == 0;
         try
         {
-            handler.BeginReconnect(
-                lastSession,
-                (int)_options.ReconnectPeriod.TotalMilliseconds,
-                OnReconnectComplete);
-        }
-        catch
-        {
-            // If the SDK refuses to re-arm (e.g. the last session is fully torn down), drop
-            // the handler so a later operator ReinitializeAsync isn't blocked by a stale one.
-            lock (_probeLock)
+            if (!Current()) return;
+            var refs = _awaitingRecovery.Keys.ToList();
+            if (refs.Count > 0)
             {
-                if (ReferenceEquals(_reconnectHandler, handler))
+                var (results, transportOk) = await ReadCoreAsync(refs, CancellationToken.None, session).ConfigureAwait(false);
+                if (!transportOk)
                 {
-                    _reconnectHandler.Dispose();
-                    _reconnectHandler = null;
+                    // Re-arm only for the session this recovery belongs to.
+                    if (ReferenceEquals(session, Session)) Interlocked.Exchange(ref _sessionLost, 1);
+                    return;
+                }
+                for (var i = 0; i < refs.Count; i++)
+                {
+                    if (!Current()) return;
+                    if (!_awaitingRecovery.TryRemove(refs[i], out _)) continue; // a notification got there first
+                    _lastValues[refs[i]] = results[i];
+                    foreach (var rs in _subscriptions.Values)
+                        if (rs.ItemHandlers.Any(p => RefOf(p.Item) == refs[i]))
+                            RaiseDataChange(new DataChangeEventArgs(rs.Handle, refs[i], results[i]));
                 }
             }
+
+            if (!Current()) return;
+            AnnotateAlarmQuality(StatusGood, Clock.GetUtcNow().UtcDateTime);
+            if (!_alarmSubscriptions.IsEmpty)
+                await RefreshAllAlarmSubscriptionsAsync(CancellationToken.None).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': in-place recovery after an outage failed.",
+                _driverInstanceId);
+        }
+    }
+
+    /// <summary>Quality-only annotation (#626 pattern) of every subscribed alarm condition.</summary>
+    private void AnnotateAlarmQuality(uint quality, DateTime now)
+    {
+        if (_alarmSubscriptions.IsEmpty || OnAlarmEvent is null) return;
+        var handle = _alarmSubscriptions.Values.First().Handle;
+        lock (_alarmStateLock)
+        {
+            if (_lastAlarmRefSet is null) return;
+            foreach (var conditionId in _lastAlarmRefSet)
+            {
+                _alarmStateByRawPath.TryGetValue(conditionId, out var told);
+                var e = new AlarmEventArgs(handle,
+                    SourceNodeId: conditionId,
+                    ConditionId: conditionId,
+                    AlarmType: "ConditionType",
+                    Message: told.Message ?? string.Empty,
+                    Severity: told.Severity is { } sev ? MapSeverity((ushort)sev) : AlarmSeverity.Low,
+                    SourceTimestampUtc: now,
+                    Kind: AlarmTransitionKind.Unspecified,
+                    OpcUaSeverity: told.Severity,
+                    QualityStatusCode: quality);
+                try { OnAlarmEvent?.Invoke(this, e); }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': alarm subscriber faulted on a quality " +
+                        "annotation for {ConditionId}.", _driverInstanceId, conditionId);
+                }
+            }
+        }
+    }
+
+    /// <summary>Raise <see cref="OnDataChange"/>, isolating a faulting subscriber from the SDK callback thread.</summary>
+    private void RaiseDataChange(DataChangeEventArgs args)
+    {
+        try { OnDataChange?.Invoke(this, args); }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': data subscriber faulted for {Ref}.",
+                _driverInstanceId, args.FullReference);
         }
     }
 
@@ -2552,7 +2816,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
     }
 
     /// <summary>
-    ///     Synchronous disposal. Cancels the reconnect handler and detaches the keep-alive
+    ///     Synchronous disposal. Detaches the keep-alive
     ///     hook synchronously (no async work on this hot path), then fires the cert-validation
     ///     handler detach. The async session-close is intentionally skipped — it requires a
     ///     live session + network round-trip and is unsafe to block-on from a potentially
@@ -2564,12 +2828,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         if (_disposed) return;
         _disposed = true;
 
-        // Cancel any in-flight reconnect handler.
-        SessionReconnectHandler? handlerToCancel;
         lock (_probeLock)
         {
-            handlerToCancel = _reconnectHandler;
-            _reconnectHandler = null;
             // Detach keep-alive and null Session so in-flight gated callers see null
             // after their next _gate.WaitAsync — they return BadCommunicationError cleanly.
             if (_keepAliveHandler is not null && Session is not null)
@@ -2579,8 +2839,6 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
             _keepAliveHandler = null;
             Session = null;
         }
-        try { handlerToCancel?.CancelReconnect(); } catch { }
-        handlerToCancel?.Dispose();
 
         // Detach the cert-validation handler registered during InitializeAsync so the
         // CertificateValidator (which may be process-shared) doesn't hold a reference to
@@ -2603,6 +2861,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         }
         catch { /* timeout or already disposed — proceed */ }
         _gate.Dispose();
+        SdkTelemetry.Release(_sdkTelemetry);
     }
 
     /// <summary>Asynchronously disposes the driver and releases all associated resources.</summary>
@@ -2631,5 +2890,6 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         }
         catch { /* timeout or already disposed */ }
         _gate.Dispose();
+        SdkTelemetry.Release(_sdkTelemetry);
     }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/SdkTelemetry.cs

@@ -1,16 +1,101 @@
+using Microsoft.Extensions.Logging;
 using Opc.Ua;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
 
 /// <summary>
-///     The one <see cref="ITelemetryContext" /> the driver hands the SDK's non-obsolete (telemetry-taking)
-///     overloads. An empty builder keeps SDK internals off any logging provider (mirrors Client.Shared's
-///     <c>ClientTelemetry</c>).
+///     The <see cref="ITelemetryContext" /> the driver hands the SDK's non-obsolete (telemetry-taking)
+///     overloads. <see cref="Context"/> has no logging provider (static helpers and tests);
+///     <see cref="ForLogger"/> routes the SDK's own Warning-and-above output (keep-alive, publish and
+///     channel errors) to a driver instance's logger — without it every SDK-side session error was
+///     invisible (Gitea #670).
 /// </summary>
 internal static class SdkTelemetry
 {
     private static readonly Lazy<ITelemetryContext> Instance = new(() => DefaultTelemetry.Create(_ => { }));
 
-    /// <summary>Gets the shared telemetry context.</summary>
+    /// <summary>Gets the shared, provider-less telemetry context.</summary>
     public static ITelemetryContext Context => Instance.Value;
+
+    /// <summary>
+    ///     Creates a telemetry context whose SDK log output at Warning and above goes to <paramref name="logger"/>,
+    ///     rate-limited (a dead upstream makes the SDK warn on every keep-alive and publish for as long as the
+    ///     outage lasts). The context owns a <see cref="ILoggerFactory"/>: release it with <see cref="Release"/>
+    ///     when the driver is disposed — <see cref="DefaultTelemetry"/> is not itself disposable.
+    /// </summary>
+    /// <param name="logger">The driver instance's logger.</param>
+    /// <returns>The telemetry context.</returns>
+    public static ITelemetryContext ForLogger(ILogger logger) =>
+        DefaultTelemetry.Create(builder => builder
+            .SetMinimumLevel(LogLevel.Warning)
+            .AddProvider(new ForwardingLoggerProvider(new RateLimitedLogger(logger, MaxEntriesPerWindow, Window))));
+
+    /// <summary>Disposes the logger factory a <see cref="ForLogger"/> context owns. Never the shared one.</summary>
+    /// <param name="context">The context to release.</param>
+    public static void Release(ITelemetryContext context)
+    {
+        if (ReferenceEquals(context, Context)) return;
+        try { (context as TelemetryContextBase)?.LoggerFactory?.Dispose(); }
+        catch { /* best-effort */ }
+    }
+
+    /// <summary>SDK log entries forwarded per <see cref="Window"/> before the rest are dropped and counted.</summary>
+    internal const int MaxEntriesPerWindow = 20;
+
+    /// <summary>The rate-limit window.</summary>
+    internal static readonly TimeSpan Window = TimeSpan.FromMinutes(1);
+
+    /// <summary>Hands every SDK logger category the one (rate-limited) driver logger.</summary>
+    private sealed class ForwardingLoggerProvider(ILogger target) : ILoggerProvider
+    {
+        public ILogger CreateLogger(string categoryName) => target;
+
+        public void Dispose() { }
+    }
+
+    /// <summary>
+    ///     Forwards at most <c>max</c> entries per window; the first entry of the next window carries a note of how
+    ///     many were suppressed.
+    /// </summary>
+    internal sealed class RateLimitedLogger(ILogger target, int max, TimeSpan window) : ILogger
+    {
+        private readonly object _lock = new();
+        private DateTime _windowStart = DateTime.MinValue;
+        private int _count;
+        private int _suppressed;
+
+        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => target.BeginScope(state);
+
+        public bool IsEnabled(LogLevel logLevel) => target.IsEnabled(logLevel);
+
+        public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
+            Func<TState, Exception?, string> formatter)
+        {
+            int suppressedNote;
+            lock (_lock)
+            {
+                var now = DateTime.UtcNow;
+                if (now - _windowStart >= window)
+                {
+                    suppressedNote = _suppressed;
+                    _windowStart = now;
+                    _count = 0;
+                    _suppressed = 0;
+                }
+                else suppressedNote = 0;
+
+                if (_count >= max)
+                {
+                    _suppressed++;
+                    return;
+                }
+                _count++;
+            }
+
+            if (suppressedNote > 0)
+                target.LogWarning("OPC UA SDK: {Suppressed} further log entries were suppressed in the last {Window}.",
+                    suppressedNote, window);
+            target.Log(logLevel, eventId, state, exception, formatter);
+        }
+    }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.csproj

@@ -31,6 +31,8 @@
         <InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests"/>
         <!-- Gitea #656: the native-alarm re-statement end-to-end tests drive this driver into the real DriverHostActor. -->
         <InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Runtime.Tests"/>
+        <!-- Gitea #670 review M1: the wire test deletes the driver's subscription server-side through its own session. -->
+        <InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests"/>
     </ItemGroup>
 
 </Project>

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs

@@ -579,8 +579,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
     ///     came back — live-confirmed against a stopped Modbus fixture, which needed a node restart. Both
     ///     comms-loss quality paths (#477 native conditions, #481 scripted-alarm inputs) hang off
     ///     <see cref="ConnectivityChanged"/>, so both were wired to a signal that never fired.
-    ///     <c>OpcUaClientDriver</c>'s own reconnect-exhausted branch already assumed this mechanism existed
-    ///     ("the Core never learns it must offer an operator reinitialize") — it did not.
+    ///     Since Gitea #670 this is also <c>OpcUaClientDriver</c>'s only recovery path for a lost upstream
+    ///     session (its in-driver SDK reconnect handler was deleted).
     ///     </para>
     ///     <para>
     ///     <b>Why the trip condition is a conjunction, and NOT simply "non-Healthy for N polls".</b>