fix(host): make a failed OPC UA SDK start visible and stop the SDK on StopAsync

A failed SDK start (port in use, PKI unwritable, bad OpcUa:* override) was one Error line: the
node stayed /health/ready 200, joined its mesh, could be elected driver Primary and acked deploys
through the Null sink while every client got connection-refused.

- IOpcUaEndpointState / OpcUaEndpointState: the start outcome is a fact the hosted service
  maintains (NotStarted / Started / Failed / Stopped + reason).
- OpcUaEndpointHealthCheck ("opcua-endpoint", Ready + Active) is Unhealthy unless Started;
  registered by AddOtOpcUaHealth(hasAdmin, hasDriver) on driver-role nodes.
- A driver-only node rethrows the start failure (Program.cs's fatal handler exits non-zero so
  the supervisor restarts it); a fused node keeps its AdminUI up but is visibly not ready.
- StopAsync detaches the sinks and then STOPS the SDK (the port closes, clients get
  connection-refused rather than a stale, inert endpoint); DisposeAsync is idempotent with it.
- The ServiceLevel calculator / redundancy election live in Cluster + Runtime (not this lane);
  the fact is exposed for the Wave 3 follow-up to consume.

Tests: port-in-use ⇒ state Failed, probe Unhealthy, real /health/ready and /health/active 503
(positive control: 200 once Started); driver-only rethrows; StopAsync closes the port and
DisposeAsync twice is a no-op; the probe registration is on both tiers for driver nodes only.

Refs #733

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs

@@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication;
 using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
 
 namespace ZB.MOM.WW.OtOpcUa.Host.Health;
 
@@ -28,11 +29,29 @@ public static class HealthEndpoints
     /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to
     /// <c>true</c> so admin/fused callers (including the test harnesses) keep the probe unchanged.
     /// </param>
+    /// <param name="hasDriver">
+    /// Whether this node carries the <c>driver</c> role and therefore serves an OPC UA endpoint. When
+    /// <c>true</c> the <see cref="OpcUaEndpointHealthCheck"/> joins the ready + active tiers (R03:2.1), reading
+    /// the <see cref="IOpcUaEndpointState"/> the host registers alongside <c>OtOpcUaServerHostedService</c>.
+    /// Defaults to <c>false</c> so admin-only callers and the existing harnesses register nothing new.
+    /// </param>
     /// <returns>The same service collection, for chaining.</returns>
-    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true)
+    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false)
     {
         var checks = services.AddHealthChecks();
 
+        // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of
+        // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or
+        // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe.
+        if (hasDriver)
+        {
+            checks.Add(new HealthCheckRegistration(
+                OpcUaEndpointHealthCheck.Name,
+                sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService<IOpcUaEndpointState>()),
+                failureStatus: null,
+                tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active }));
+        }
+
         // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config
         // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so
         // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged.

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs

@@ -0,0 +1,45 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Health;
+
+/// <summary>
+///     Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is
+///     listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy
+///     one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets
+///     connection-refused.
+/// </summary>
+/// <remarks>
+///     Registered on driver-role nodes only (<see cref="HealthEndpoints.AddOtOpcUaHealth"/>); an
+///     admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts.
+/// </remarks>
+public sealed class OpcUaEndpointHealthCheck : IHealthCheck
+{
+    /// <summary>The registration name, as it appears in the health JSON.</summary>
+    public const string Name = "opcua-endpoint";
+
+    private readonly IOpcUaEndpointState _state;
+
+    /// <summary>Initializes a new instance of the <see cref="OpcUaEndpointHealthCheck"/> class.</summary>
+    /// <param name="state">The endpoint lifecycle fact the hosted service maintains.</param>
+    public OpcUaEndpointHealthCheck(IOpcUaEndpointState state)
+    {
+        ArgumentNullException.ThrowIfNull(state);
+        _state = state;
+    }
+
+    /// <inheritdoc />
+    public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+    {
+        var data = new Dictionary<string, object> { ["status"] = _state.Status.ToString() };
+        var result = _state.Status switch
+        {
+            OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data),
+            OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data),
+            OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data),
+            _ => HealthCheckResult.Unhealthy(
+                "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data),
+        };
+        return Task.FromResult(result);
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs

@@ -0,0 +1,73 @@
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>Lifecycle of this node's OPC UA endpoint as <see cref="OtOpcUaServerHostedService"/> drives it.</summary>
+public enum OpcUaEndpointStatus
+{
+    /// <summary>The hosted service has not run <c>StartAsync</c> yet.</summary>
+    NotStarted,
+
+    /// <summary>The SDK is listening and the address-space / ServiceLevel sinks are bound.</summary>
+    Started,
+
+    /// <summary>The SDK failed to start (port in use, PKI unwritable, bad <c>OpcUa:*</c> override, …).</summary>
+    Failed,
+
+    /// <summary>The hosted service stopped the SDK; the port is released.</summary>
+    Stopped,
+}
+
+/// <summary>
+///     The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed
+///     start was one Error line: <c>/health/ready</c> stayed 200, the node joined its mesh, could be
+///     elected driver Primary and acked deploys as Applied through the Null sink — while clients got
+///     connection-refused from the node the fleet called healthy.
+/// </summary>
+/// <remarks>
+///     Consumed by <c>OpcUaEndpointHealthCheck</c> (Ready + Active tiers). The redundancy election and the
+///     ServiceLevel calculator do not read it yet — they live outside the Host (<c>Cluster</c> /
+///     <c>Runtime</c>); feeding them is the Wave 3 follow-up the lane report names. Until then a fused
+///     node whose SDK failed is visible (503) but still Primary-eligible.
+/// </remarks>
+public interface IOpcUaEndpointState
+{
+    /// <summary>The endpoint's current lifecycle status.</summary>
+    OpcUaEndpointStatus Status { get; }
+
+    /// <summary>Why the start failed, when <see cref="Status"/> is <see cref="OpcUaEndpointStatus.Failed"/>.</summary>
+    string? FailureReason { get; }
+}
+
+/// <summary>Mutable <see cref="IOpcUaEndpointState"/>; written by the hosted service, read by the health check.</summary>
+public sealed class OpcUaEndpointState : IOpcUaEndpointState
+{
+    private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted;
+    private volatile string? _failureReason;
+
+    /// <inheritdoc />
+    public OpcUaEndpointStatus Status => _status;
+
+    /// <inheritdoc />
+    public string? FailureReason => _failureReason;
+
+    /// <summary>Records a successful SDK start.</summary>
+    public void MarkStarted()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Started;
+    }
+
+    /// <summary>Records a failed SDK start.</summary>
+    /// <param name="reason">The failure, as the health probe should report it.</param>
+    public void MarkFailed(string reason)
+    {
+        _failureReason = reason;
+        _status = OpcUaEndpointStatus.Failed;
+    }
+
+    /// <summary>Records that the hosted service stopped the SDK.</summary>
+    public void MarkStopped()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Stopped;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Core.Authorization;
@@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
     private readonly HistorianUpdaterHolder _historianUpdater;
     private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
+    private readonly OpcUaEndpointState _endpointState;
+    private readonly bool _rethrowStartFailure;
 
     private OpcUaApplicationHost? _appHost;
     private OtOpcUaSdkServer? _server;
@@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     /// <param name="historyUpdateRoleView">The HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
     /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
     /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
+    /// <param name="endpointState">The endpoint lifecycle fact this service maintains for the
+    /// <c>opcua-endpoint</c> health probe (R03:2.1). Optional: a private instance is kept when none is registered,
+    /// so a harness that never probes health is unchanged.</param>
+    /// <param name="roles">The node's resolved roles. A node with the driver role and NO admin role exists only
+    /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and
+    /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through
+    /// the health probe instead. Null (harnesses) behaves as fused.</param>
     public OtOpcUaServerHostedService(
         IOptions<OpcUaApplicationHostOptions> options,
         DeferredAddressSpaceSink deferredSink,
@@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         ILoggerFactory loggerFactory,
         IDriverPrimaryRoleSource? primaryRoleSource = null,
         HistorianUpdaterHolder? historianUpdater = null,
-        IHistoryUpdateRoleView? historyUpdateRoleView = null)
+        IHistoryUpdateRoleView? historyUpdateRoleView = null,
+        OpcUaEndpointState? endpointState = null,
+        EffectiveRoles? roles = null)
     {
         ArgumentNullException.ThrowIfNull(aclOptions);
         ArgumentNullException.ThrowIfNull(time);
@@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         _primaryRoleSource = primaryRoleSource;
         _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
         _historyUpdateRoleView = historyUpdateRoleView;
+        _endpointState = endpointState ?? new OpcUaEndpointState();
+        _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false };
         _logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
     }
 
@@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         }
         catch (Exception ex)
         {
+            // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy
+            // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits
+            // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct
+            // the OpcUa:* config without a process bounce — but it is visibly not ready.
+            _endpointState.MarkFailed(ex.Message);
+            await ReleaseFailedStartAsync().ConfigureAwait(false);
+            if (_rethrowStartFailure)
+            {
+                _logger.LogCritical(ex,
+                    "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " +
+                    "node justifies staying up; failing host start so the supervisor restarts the process");
+                throw;
+            }
             _logger.LogError(ex,
-                "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op");
-            // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot.
-            // Operators see the failure via the logs + can correct config without a process bounce
-            // of the whole binary.
+                "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " +
+                "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section");
             return;
         }
 
@@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         {
             _logger.LogWarning(
                 "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null");
+            _endpointState.MarkFailed("SDK reported started but the node manager is null");
             return;
         }
 
@@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         {
             _logger.LogError(ex,
                 "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
+            _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message);
             try
             {
-                await StopAsync(cancellationToken).ConfigureAwait(false);
+                DetachSinks();
             }
             catch (Exception cleanupError)
             {
                 _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
             }
-            var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
-                async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
-                () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
-            if (released)
-            {
-                _appHost = null;
-                _server = null;
-            }
+            await ReleaseEndpointAsync().ConfigureAwait(false);
             return;
         }
 
@@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         // attached, so the first tick cannot land on a half-wired server.
         StartRevalidationTimer();
 
+        _endpointState.MarkStarted();
         _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
     }
 
+    /// <summary>
+    ///     Releases an SDK whose <c>StartAsync</c> threw. The application instance may hold a half-opened
+    ///     listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the
+    ///     transport is up); disposing it is the only way to be sure. The state stays Failed.
+    /// </summary>
+    private async Task ReleaseFailedStartAsync()
+    {
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is null) return;
+        try
+        {
+            await appHost.DisposeAsync().ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw");
+        }
+    }
+
+    /// <summary>
+    ///     Stops the SDK and disposes the application host, releasing endpoint ownership only when both
+    ///     succeeded (<see cref="FailedServiceLevelBindingCleanup.StopAndDisposeAsync"/>): on a failure the
+    ///     fields are retained so <see cref="DisposeAsync"/> retries at host disposal.
+    /// </summary>
+    private async Task ReleaseEndpointAsync()
+    {
+        var server = _server;
+        var appHost = _appHost;
+        if (server is null && appHost is null) return;
+        var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
+            async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); },
+            () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask,
+            _logger).ConfigureAwait(false);
+        if (released)
+        {
+            _appHost = null;
+            _server = null;
+        }
+    }
+
     /// <summary>
     ///     The <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
     ///     (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
@@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     }
 
     /// <summary>
-    /// Stops the OPC UA server asynchronously.
+    /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes.
     /// </summary>
+    /// <remarks>
+    ///     R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown,
+    ///     the other hosted services' StopAsync), serving the last-written values with an emptied authority:
+    ///     reads succeeded on stale data while writes got "unavailable", and clients saw a server that was
+    ///     up but inert instead of the connection-refused that triggers their failover. <see cref="DisposeAsync"/>
+    ///     stays idempotent with this: ownership is nulled once the stop succeeded.
+    /// </remarks>
     /// <param name="cancellationToken">Cancellation token.</param>
     /// <returns>A task that represents the asynchronous operation.</returns>
-    public Task StopAsync(CancellationToken cancellationToken)
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        DetachSinks();
+        await ReleaseEndpointAsync().ConfigureAwait(false);
+        if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped();
+    }
+
+    /// <summary>
+    ///     Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor
+    ///     or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running
+    ///     until <see cref="ReleaseEndpointAsync"/>.
+    /// </summary>
+    private void DetachSinks()
     {
         // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
         // authority and gate have already been cleared is the shape this whole teardown block exists to
@@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         // request rather than let the seams fall back to their role checks — a stopping node should shed
         // enforcement in the same order it acquired it.
         _server?.SetDataPlaneAuthorizer(null);
-        return Task.CompletedTask;
     }
 
     /// <summary>
@@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     public async ValueTask DisposeAsync()
     {
         // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
-        // not leave a tick armed against a server that is about to go away.
+        // not leave a tick armed against a server that is about to go away. A StopAsync that released the
+        // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left
+        // them set, and this is the retry.
         _revalidationTimer?.Dispose();
         _revalidationTimer = null;
 
-        if (_appHost is not null) await _appHost.DisposeAsync();
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is not null) await appHost.DisposeAsync();
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -452,6 +452,11 @@ if (hasDriver)
     // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
     // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
     builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
+    // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint
+    // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the
+    // interface, so the writer and the probe cannot disagree.
+    builder.Services.AddSingleton<OpcUaEndpointState>();
+    builder.Services.AddSingleton<IOpcUaEndpointState>(sp => sp.GetRequiredService<OpcUaEndpointState>());
     builder.Services.AddHostedService<OtOpcUaServerHostedService>();
 }
 
@@ -554,8 +559,9 @@ if (hasAdmin)
 builder.Services.AddOtOpcUaSecrets(builder.Configuration);
 
 // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
-// probing it would throw when the readiness endpoint resolves the factory.
-builder.Services.AddOtOpcUaHealth(hasAdmin);
+// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the
+// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green.
+builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver);
 builder.Services.AddOtOpcUaObservability(builder.Configuration);
 
 // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,