fix(host): make a failed OPC UA SDK start visible and stop the SDK on StopAsync

A failed SDK start (port in use, PKI unwritable, bad OpcUa:* override) was one Error line: the node stayed /health/ready 200, joined its mesh, could be elected driver Primary and acked deploys through the Null sink while every client got connection-refused. - IOpcUaEndpointState / OpcUaEndpointState: the start outcome is a fact the hosted service maintains (NotStarted / Started / Failed / Stopped + reason). - OpcUaEndpointHealthCheck ("opcua-endpoint", Ready + Active) is Unhealthy unless Started; registered by AddOtOpcUaHealth(hasAdmin, hasDriver) on driver-role nodes. - A driver-only node rethrows the start failure (Program.cs's fatal handler exits non-zero so the supervisor restarts it); a fused node keeps its AdminUI up but is visibly not ready. - StopAsync detaches the sinks and then STOPS the SDK (the port closes, clients get connection-refused rather than a stale, inert endpoint); DisposeAsync is idempotent with it. - The ServiceLevel calculator / redundancy election live in Cluster + Runtime (not this lane); the fact is exposed for the Wave 3 follow-up to consume. Tests: port-in-use ⇒ state Failed, probe Unhealthy, real /health/ready and /health/active 503 (positive control: 200 once Started); driver-only rethrows; StopAsync closes the port and DisposeAsync twice is a no-op; the probe registration is on both tiers for driver nodes only. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:15:05 -04:00
parent ae84253773
commit f5579e6ef2
7 changed files with 492 additions and 22 deletions
@@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication;
 using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;

 namespace ZB.MOM.WW.OtOpcUa.Host.Health;

@@ -28,11 +29,29 @@ public static class HealthEndpoints
    /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to
    /// <c>true</c> so admin/fused callers (including the test harnesses) keep the probe unchanged.
    /// </param>
+    /// <param name="hasDriver">
+    /// Whether this node carries the <c>driver</c> role and therefore serves an OPC UA endpoint. When
+    /// <c>true</c> the <see cref="OpcUaEndpointHealthCheck"/> joins the ready + active tiers (R03:2.1), reading
+    /// the <see cref="IOpcUaEndpointState"/> the host registers alongside <c>OtOpcUaServerHostedService</c>.
+    /// Defaults to <c>false</c> so admin-only callers and the existing harnesses register nothing new.
+    /// </param>
    /// <returns>The same service collection, for chaining.</returns>
-    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true)
+    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false)
    {
        var checks = services.AddHealthChecks();

+        // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of
+        // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or
+        // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe.
+        if (hasDriver)
+        {
+            checks.Add(new HealthCheckRegistration(
+                OpcUaEndpointHealthCheck.Name,
+                sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService<IOpcUaEndpointState>()),
+                failureStatus: null,
+                tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active }));
+        }
+
        // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config
        // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so
        // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged.
@@ -0,0 +1,45 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Health;
+
+/// <summary>
+///     Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is
+///     listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy
+///     one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets
+///     connection-refused.
+/// </summary>
+/// <remarks>
+///     Registered on driver-role nodes only (<see cref="HealthEndpoints.AddOtOpcUaHealth"/>); an
+///     admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts.
+/// </remarks>
+public sealed class OpcUaEndpointHealthCheck : IHealthCheck
+{
+    /// <summary>The registration name, as it appears in the health JSON.</summary>
+    public const string Name = "opcua-endpoint";
+
+    private readonly IOpcUaEndpointState _state;
+
+    /// <summary>Initializes a new instance of the <see cref="OpcUaEndpointHealthCheck"/> class.</summary>
+    /// <param name="state">The endpoint lifecycle fact the hosted service maintains.</param>
+    public OpcUaEndpointHealthCheck(IOpcUaEndpointState state)
+    {
+        ArgumentNullException.ThrowIfNull(state);
+        _state = state;
+    }
+
+    /// <inheritdoc />
+    public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+    {
+        var data = new Dictionary<string, object> { ["status"] = _state.Status.ToString() };
+        var result = _state.Status switch
+        {
+            OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data),
+            OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data),
+            OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data),
+            _ => HealthCheckResult.Unhealthy(
+                "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data),
+        };
+        return Task.FromResult(result);
+    }
+}
@@ -0,0 +1,73 @@
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>Lifecycle of this node's OPC UA endpoint as <see cref="OtOpcUaServerHostedService"/> drives it.</summary>
+public enum OpcUaEndpointStatus
+{
+    /// <summary>The hosted service has not run <c>StartAsync</c> yet.</summary>
+    NotStarted,
+
+    /// <summary>The SDK is listening and the address-space / ServiceLevel sinks are bound.</summary>
+    Started,
+
+    /// <summary>The SDK failed to start (port in use, PKI unwritable, bad <c>OpcUa:*</c> override, …).</summary>
+    Failed,
+
+    /// <summary>The hosted service stopped the SDK; the port is released.</summary>
+    Stopped,
+}
+
+/// <summary>
+///     The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed
+///     start was one Error line: <c>/health/ready</c> stayed 200, the node joined its mesh, could be
+///     elected driver Primary and acked deploys as Applied through the Null sink — while clients got
+///     connection-refused from the node the fleet called healthy.
+/// </summary>
+/// <remarks>
+///     Consumed by <c>OpcUaEndpointHealthCheck</c> (Ready + Active tiers). The redundancy election and the
+///     ServiceLevel calculator do not read it yet — they live outside the Host (<c>Cluster</c> /
+///     <c>Runtime</c>); feeding them is the Wave 3 follow-up the lane report names. Until then a fused
+///     node whose SDK failed is visible (503) but still Primary-eligible.
+/// </remarks>
+public interface IOpcUaEndpointState
+{
+    /// <summary>The endpoint's current lifecycle status.</summary>
+    OpcUaEndpointStatus Status { get; }
+
+    /// <summary>Why the start failed, when <see cref="Status"/> is <see cref="OpcUaEndpointStatus.Failed"/>.</summary>
+    string? FailureReason { get; }
+}
+
+/// <summary>Mutable <see cref="IOpcUaEndpointState"/>; written by the hosted service, read by the health check.</summary>
+public sealed class OpcUaEndpointState : IOpcUaEndpointState
+{
+    private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted;
+    private volatile string? _failureReason;
+
+    /// <inheritdoc />
+    public OpcUaEndpointStatus Status => _status;
+
+    /// <inheritdoc />
+    public string? FailureReason => _failureReason;
+
+    /// <summary>Records a successful SDK start.</summary>
+    public void MarkStarted()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Started;
+    }
+
+    /// <summary>Records a failed SDK start.</summary>
+    /// <param name="reason">The failure, as the health probe should report it.</param>
+    public void MarkFailed(string reason)
+    {
+        _failureReason = reason;
+        _status = OpcUaEndpointStatus.Failed;
+    }
+
+    /// <summary>Records that the hosted service stopped the SDK.</summary>
+    public void MarkStopped()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Stopped;
+    }
+}
@@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Core.Authorization;
@@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
    private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
    private readonly HistorianUpdaterHolder _historianUpdater;
    private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
+    private readonly OpcUaEndpointState _endpointState;
+    private readonly bool _rethrowStartFailure;

    private OpcUaApplicationHost? _appHost;
    private OtOpcUaSdkServer? _server;
@@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
    /// <param name="historyUpdateRoleView">The HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
    /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
    /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
+    /// <param name="endpointState">The endpoint lifecycle fact this service maintains for the
+    /// <c>opcua-endpoint</c> health probe (R03:2.1). Optional: a private instance is kept when none is registered,
+    /// so a harness that never probes health is unchanged.</param>
+    /// <param name="roles">The node's resolved roles. A node with the driver role and NO admin role exists only
+    /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and
+    /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through
+    /// the health probe instead. Null (harnesses) behaves as fused.</param>
    public OtOpcUaServerHostedService(
        IOptions<OpcUaApplicationHostOptions> options,
        DeferredAddressSpaceSink deferredSink,
@@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        ILoggerFactory loggerFactory,
        IDriverPrimaryRoleSource? primaryRoleSource = null,
        HistorianUpdaterHolder? historianUpdater = null,
-        IHistoryUpdateRoleView? historyUpdateRoleView = null)
+        IHistoryUpdateRoleView? historyUpdateRoleView = null,
+        OpcUaEndpointState? endpointState = null,
+        EffectiveRoles? roles = null)
    {
        ArgumentNullException.ThrowIfNull(aclOptions);
        ArgumentNullException.ThrowIfNull(time);
@@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        _primaryRoleSource = primaryRoleSource;
        _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
        _historyUpdateRoleView = historyUpdateRoleView;
+        _endpointState = endpointState ?? new OpcUaEndpointState();
+        _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false };
        _logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
    }

@@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        }
        catch (Exception ex)
        {
+            // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy
+            // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits
+            // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct
+            // the OpcUa:* config without a process bounce — but it is visibly not ready.
+            _endpointState.MarkFailed(ex.Message);
+            await ReleaseFailedStartAsync().ConfigureAwait(false);
+            if (_rethrowStartFailure)
+            {
+                _logger.LogCritical(ex,
+                    "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " +
+                    "node justifies staying up; failing host start so the supervisor restarts the process");
+                throw;
+            }
            _logger.LogError(ex,
-                "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op");
-            // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot.
-            // Operators see the failure via the logs + can correct config without a process bounce
-            // of the whole binary.
+                "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " +
+                "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section");
            return;
        }

@@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        {
            _logger.LogWarning(
                "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null");
+            _endpointState.MarkFailed("SDK reported started but the node manager is null");
            return;
        }

@@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        {
            _logger.LogError(ex,
                "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
+            _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message);
            try
            {
-                await StopAsync(cancellationToken).ConfigureAwait(false);
+                DetachSinks();
            }
            catch (Exception cleanupError)
            {
                _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
            }
-            var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
-                async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
-                () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
-            if (released)
-            {
-                _appHost = null;
-                _server = null;
-            }
+            await ReleaseEndpointAsync().ConfigureAwait(false);
            return;
        }

@@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        // attached, so the first tick cannot land on a half-wired server.
        StartRevalidationTimer();

+        _endpointState.MarkStarted();
        _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
    }

+    /// <summary>
+    ///     Releases an SDK whose <c>StartAsync</c> threw. The application instance may hold a half-opened
+    ///     listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the
+    ///     transport is up); disposing it is the only way to be sure. The state stays Failed.
+    /// </summary>
+    private async Task ReleaseFailedStartAsync()
+    {
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is null) return;
+        try
+        {
+            await appHost.DisposeAsync().ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw");
+        }
+    }
+
+    /// <summary>
+    ///     Stops the SDK and disposes the application host, releasing endpoint ownership only when both
+    ///     succeeded (<see cref="FailedServiceLevelBindingCleanup.StopAndDisposeAsync"/>): on a failure the
+    ///     fields are retained so <see cref="DisposeAsync"/> retries at host disposal.
+    /// </summary>
+    private async Task ReleaseEndpointAsync()
+    {
+        var server = _server;
+        var appHost = _appHost;
+        if (server is null && appHost is null) return;
+        var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
+            async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); },
+            () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask,
+            _logger).ConfigureAwait(false);
+        if (released)
+        {
+            _appHost = null;
+            _server = null;
+        }
+    }
+
    /// <summary>
    ///     The <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
    ///     (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
@@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
    }

    /// <summary>
-    /// Stops the OPC UA server asynchronously.
+    /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes.
    /// </summary>
+    /// <remarks>
+    ///     R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown,
+    ///     the other hosted services' StopAsync), serving the last-written values with an emptied authority:
+    ///     reads succeeded on stale data while writes got "unavailable", and clients saw a server that was
+    ///     up but inert instead of the connection-refused that triggers their failover. <see cref="DisposeAsync"/>
+    ///     stays idempotent with this: ownership is nulled once the stop succeeded.
+    /// </remarks>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
-    public Task StopAsync(CancellationToken cancellationToken)
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        DetachSinks();
+        await ReleaseEndpointAsync().ConfigureAwait(false);
+        if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped();
+    }
+
+    /// <summary>
+    ///     Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor
+    ///     or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running
+    ///     until <see cref="ReleaseEndpointAsync"/>.
+    /// </summary>
+    private void DetachSinks()
    {
        // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
        // authority and gate have already been cleared is the shape this whole teardown block exists to
@@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
        // request rather than let the seams fall back to their role checks — a stopping node should shed
        // enforcement in the same order it acquired it.
        _server?.SetDataPlaneAuthorizer(null);
-        return Task.CompletedTask;
    }

    /// <summary>
@@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
    public async ValueTask DisposeAsync()
    {
        // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
-        // not leave a tick armed against a server that is about to go away.
+        // not leave a tick armed against a server that is about to go away. A StopAsync that released the
+        // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left
+        // them set, and this is the retry.
        _revalidationTimer?.Dispose();
        _revalidationTimer = null;

-        if (_appHost is not null) await _appHost.DisposeAsync();
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is not null) await appHost.DisposeAsync();
    }
 }
@@ -452,6 +452,11 @@ if (hasDriver)
    // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
    // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
    builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
+    // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint
+    // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the
+    // interface, so the writer and the probe cannot disagree.
+    builder.Services.AddSingleton<OpcUaEndpointState>();
+    builder.Services.AddSingleton<IOpcUaEndpointState>(sp => sp.GetRequiredService<OpcUaEndpointState>());
    builder.Services.AddHostedService<OtOpcUaServerHostedService>();
 }

@@ -554,8 +559,9 @@ if (hasAdmin)
 builder.Services.AddOtOpcUaSecrets(builder.Configuration);

 // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
-// probing it would throw when the readiness endpoint resolves the factory.
-builder.Services.AddOtOpcUaHealth(hasAdmin);
+// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the
+// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green.
+builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver);
 builder.Services.AddOtOpcUaObservability(builder.Configuration);

 // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,
@@ -49,6 +49,34 @@ public sealed class ActiveTierRegistrationTests
        ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active);
    }

+    /// <summary>
+    ///     R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not
+    ///     be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by.
+    /// </summary>
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver)
+    {
+        var registrations = new ServiceCollection()
+            .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver)
+            .BuildServiceProvider()
+            .GetRequiredService<IOptions<HealthCheckServiceOptions>>()
+            .Value.Registrations
+            .Where(r => r.Name == OpcUaEndpointHealthCheck.Name)
+            .ToList();
+
+        if (!hasDriver)
+        {
+            registrations.ShouldBeEmpty();
+            return;
+        }
+
+        var registration = registrations.ShouldHaveSingleItem();
+        registration.Tags.ShouldContain(ZbHealthTags.Ready);
+        registration.Tags.ShouldContain(ZbHealthTags.Active);
+    }
+
    [Fact]
    public void Active_tier_check_is_not_on_the_ready_tier()
    {
@@ -0,0 +1,213 @@
+using System.Net;
+using System.Net.Sockets;
+using Akka.Hosting;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using ZB.MOM.WW.OtOpcUa.Host.Health;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
+
+/// <summary>
+///     R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined
+///     its mesh, and acked deploys through the Null sink while clients got connection-refused. The start
+///     outcome is now a fact (<see cref="IOpcUaEndpointState"/>) the ready/active tiers read, a driver-only
+///     node rethrows so the process exits, and <c>StopAsync</c> actually stops the SDK.
+/// </summary>
+public sealed class OtOpcUaServerHostedServiceStartFailureTests
+{
+    private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    ///     Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the
+    ///     real <c>/health/ready</c> endpoint answers 503 — while StartAsync itself returns (fused node).
+    ///     Positive control: the same endpoint answers 200 once the state is Started.
+    /// </summary>
+    [Fact]
+    public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503()
+    {
+        using var deadline = new CancellationTokenSource(Deadline);
+        using var squatter = SquatAllInterfaces(out var port);
+        var pki = NewPki();
+        var state = new OpcUaEndpointState();
+        var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null);
+
+        var service = NewService(port, pki, state, fused);
+        try
+        {
+            await service.StartAsync(deadline.Token);   // must NOT throw on a fused node
+
+            state.Status.ShouldBe(OpcUaEndpointStatus.Failed);
+            state.FailureReason.ShouldNotBeNullOrWhiteSpace();
+
+            var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext());
+            probe.Status.ShouldBe(HealthStatus.Unhealthy);
+            probe.Description.ShouldNotBeNull();
+            probe.Description.ShouldContain("failed to start");
+
+            await using var web = BuildHealthHost(state);
+            await web.StartAsync(deadline.Token);
+            using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) };
+
+            (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
+            (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
+
+            // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the
+            // 503 above measured the probe and not a broken health pipeline.
+            state.MarkStarted();
+            (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK);
+            await web.StopAsync(deadline.Token);
+        }
+        finally
+        {
+            await service.DisposeAsync();
+            squatter.Stop();
+            Cleanup(pki);
+        }
+    }
+
+    /// <summary>A driver-only node has nothing else to stay up for: the failed start propagates.</summary>
+    [Fact]
+    public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start()
+    {
+        using var deadline = new CancellationTokenSource(Deadline);
+        using var squatter = SquatAllInterfaces(out var port);
+        var pki = NewPki();
+        var state = new OpcUaEndpointState();
+        var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null);
+
+        var service = NewService(port, pki, state, driverOnly);
+        try
+        {
+            await Should.ThrowAsync<Exception>(() => service.StartAsync(deadline.Token));
+            state.Status.ShouldBe(OpcUaEndpointStatus.Failed);
+        }
+        finally
+        {
+            await service.DisposeAsync();
+            squatter.Stop();
+            Cleanup(pki);
+        }
+    }
+
+    /// <summary>
+    ///     A clean start is Started and listening; <c>StopAsync</c> closes the port (connection refused, not
+    ///     a stale-but-listening endpoint) and marks Stopped; <c>DisposeAsync</c> twice afterwards is a no-op.
+    /// </summary>
+    [Fact]
+    public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent()
+    {
+        using var deadline = new CancellationTokenSource(Deadline);
+        var port = FreePort();
+        var pki = NewPki();
+        var state = new OpcUaEndpointState();
+        var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null));
+        try
+        {
+            await service.StartAsync(deadline.Token);
+            state.Status.ShouldBe(OpcUaEndpointStatus.Started);
+            using (var client = new TcpClient())
+            {
+                await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token);
+                client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync");
+            }
+
+            await service.StopAsync(deadline.Token);
+
+            state.Status.ShouldBe(OpcUaEndpointStatus.Stopped);
+            using (var client = new TcpClient())
+            {
+                await Should.ThrowAsync<SocketException>(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token));
+            }
+
+            await service.DisposeAsync();
+            await service.DisposeAsync();   // idempotent
+        }
+        finally
+        {
+            await service.DisposeAsync();
+            Cleanup(pki);
+        }
+    }
+
+    private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) =>
+        new(
+            Options.Create(new OpcUaApplicationHostOptions
+            {
+                ApplicationName = "OtOpcUa.StartFailure",
+                ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}",
+                OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki,
+            }),
+            new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(),
+            NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance,
+            EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(),
+            () => throw new NotSupportedException("no actor system required"), new ActorRegistry(),
+            null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance,
+            endpointState: state, roles: roles);
+
+    /// <summary>The production health registration (driver role) over the supplied state, on a loopback Kestrel.</summary>
+    private static WebApplication BuildHealthHost(OpcUaEndpointState state)
+    {
+        var builder = WebApplication.CreateBuilder();
+        builder.WebHost.UseUrls("http://127.0.0.1:0");
+        builder.Logging.ClearProviders();
+        builder.Services.AddSingleton<IOpcUaEndpointState>(state);
+        builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true);
+        // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so
+        // the status measured is the endpoint's. (The registration shape itself is pinned separately.)
+        builder.Services.Configure<HealthCheckServiceOptions>(o =>
+        {
+            foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList())
+                o.Registrations.Remove(r);
+        });
+        var app = builder.Build();
+        app.MapOtOpcUaHealth();
+        return app;
+    }
+
+    /// <summary>
+    ///     Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the
+    ///     wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both
+    ///     sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of
+    ///     this test proved exactly that. The SDK also only fails when NEITHER family binds.
+    /// </summary>
+    private static TcpListener SquatAllInterfaces(out int port)
+    {
+        var squatter = new TcpListener(IPAddress.IPv6Any, 0);
+        squatter.Server.DualMode = true;
+        squatter.ExclusiveAddressUse = true;
+        squatter.Start();
+        port = ((IPEndPoint)squatter.LocalEndpoint).Port;
+        return squatter;
+    }
+
+    private static int FreePort()
+    {
+        using var probe = new TcpListener(IPAddress.Loopback, 0);
+        probe.Start();
+        var port = ((IPEndPoint)probe.LocalEndpoint).Port;
+        probe.Stop();
+        return port;
+    }
+
+    private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}");
+
+    private static void Cleanup(string pki)
+    {
+        if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true);
+    }
+}