diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index f7a740bd..2f57158f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; @@ -28,11 +29,29 @@ public static class HealthEndpoints /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to /// true so admin/fused callers (including the test harnesses) keep the probe unchanged. /// + /// + /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When + /// true the joins the ready + active tiers (R03:2.1), reading + /// the the host registers alongside OtOpcUaServerHostedService. + /// Defaults to false so admin-only callers and the existing harnesses register nothing new. + /// /// The same service collection, for chaining. - public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true) + public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false) { var checks = services.AddHealthChecks(); + // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of + // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or + // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + if (hasDriver) + { + checks.Add(new HealthCheckRegistration( + OpcUaEndpointHealthCheck.Name, + sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), + failureStatus: null, + tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + } + // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs new file mode 100644 index 00000000..8caa69f5 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Health; + +/// +/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy +/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets +/// connection-refused. +/// +/// +/// Registered on driver-role nodes only (); an +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// +public sealed class OpcUaEndpointHealthCheck : IHealthCheck +{ + /// The registration name, as it appears in the health JSON. + public const string Name = "opcua-endpoint"; + + private readonly IOpcUaEndpointState _state; + + /// Initializes a new instance of the class. + /// The endpoint lifecycle fact the hosted service maintains. + public OpcUaEndpointHealthCheck(IOpcUaEndpointState state) + { + ArgumentNullException.ThrowIfNull(state); + _state = state; + } + + /// + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var data = new Dictionary { ["status"] = _state.Status.ToString() }; + var result = _state.Status switch + { + OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data), + OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data), + OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data), + _ => HealthCheckResult.Unhealthy( + "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data), + }; + return Task.FromResult(result); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs new file mode 100644 index 00000000..c2377ea1 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs @@ -0,0 +1,73 @@ +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// Lifecycle of this node's OPC UA endpoint as drives it. +public enum OpcUaEndpointStatus +{ + /// The hosted service has not run StartAsync yet. + NotStarted, + + /// The SDK is listening and the address-space / ServiceLevel sinks are bound. + Started, + + /// The SDK failed to start (port in use, PKI unwritable, bad OpcUa:* override, …). + Failed, + + /// The hosted service stopped the SDK; the port is released. + Stopped, +} + +/// +/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed +/// start was one Error line: /health/ready stayed 200, the node joined its mesh, could be +/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got +/// connection-refused from the node the fleet called healthy. +/// +/// +/// Consumed by OpcUaEndpointHealthCheck (Ready + Active tiers). The redundancy election and the +/// ServiceLevel calculator do not read it yet — they live outside the Host (Cluster / +/// Runtime); feeding them is the Wave 3 follow-up the lane report names. Until then a fused +/// node whose SDK failed is visible (503) but still Primary-eligible. +/// +public interface IOpcUaEndpointState +{ + /// The endpoint's current lifecycle status. + OpcUaEndpointStatus Status { get; } + + /// Why the start failed, when is . + string? FailureReason { get; } +} + +/// Mutable ; written by the hosted service, read by the health check. +public sealed class OpcUaEndpointState : IOpcUaEndpointState +{ + private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted; + private volatile string? _failureReason; + + /// + public OpcUaEndpointStatus Status => _status; + + /// + public string? FailureReason => _failureReason; + + /// Records a successful SDK start. + public void MarkStarted() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Started; + } + + /// Records a failed SDK start. + /// The failure, as the health probe should report it. + public void MarkFailed(string reason) + { + _failureReason = reason; + _status = OpcUaEndpointStatus.Failed; + } + + /// Records that the hosted service stopped the SDK. + public void MarkStopped() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Stopped; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 79f62cd8..609b0087 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly IDriverPrimaryRoleSource? _primaryRoleSource; private readonly HistorianUpdaterHolder _historianUpdater; private readonly IHistoryUpdateRoleView? _historyUpdateRoleView; + private readonly OpcUaEndpointState _endpointState; + private readonly bool _rethrowStartFailure; private OpcUaApplicationHost? _appHost; private OtOpcUaSdkServer? _server; @@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// The HistoryUpdate verdict DriverHostActor publishes (the /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow). + /// The endpoint lifecycle fact this service maintains for the + /// opcua-endpoint health probe (R03:2.1). Optional: a private instance is kept when none is registered, + /// so a harness that never probes health is unchanged. + /// The node's resolved roles. A node with the driver role and NO admin role exists only + /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and + /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through + /// the health probe instead. Null (harnesses) behaves as fused. public OtOpcUaServerHostedService( IOptions options, DeferredAddressSpaceSink deferredSink, @@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl ILoggerFactory loggerFactory, IDriverPrimaryRoleSource? primaryRoleSource = null, HistorianUpdaterHolder? historianUpdater = null, - IHistoryUpdateRoleView? historyUpdateRoleView = null) + IHistoryUpdateRoleView? historyUpdateRoleView = null, + OpcUaEndpointState? endpointState = null, + EffectiveRoles? roles = null) { ArgumentNullException.ThrowIfNull(aclOptions); ArgumentNullException.ThrowIfNull(time); @@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl _primaryRoleSource = primaryRoleSource; _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None; _historyUpdateRoleView = historyUpdateRoleView; + _endpointState = endpointState ?? new OpcUaEndpointState(); + _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false }; _logger = loggerFactory.CreateLogger(); } @@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } catch (Exception ex) { + // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy + // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits + // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct + // the OpcUa:* config without a process bounce — but it is visibly not ready. + _endpointState.MarkFailed(ex.Message); + await ReleaseFailedStartAsync().ConfigureAwait(false); + if (_rethrowStartFailure) + { + _logger.LogCritical(ex, + "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " + + "node justifies staying up; failing host start so the supervisor restarts the process"); + throw; + } _logger.LogError(ex, - "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op"); - // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot. - // Operators see the failure via the logs + can correct config without a process bounce - // of the whole binary. + "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " + + "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section"); return; } @@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogWarning( "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null"); + _endpointState.MarkFailed("SDK reported started but the node manager is null"); return; } @@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogError(ex, "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues"); + _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message); try { - await StopAsync(cancellationToken).ConfigureAwait(false); + DetachSinks(); } catch (Exception cleanupError) { _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure"); } - var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( - async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false), - () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false); - if (released) - { - _appHost = null; - _server = null; - } + await ReleaseEndpointAsync().ConfigureAwait(false); return; } @@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // attached, so the first tick cannot land on a half-wired server. StartRevalidationTimer(); + _endpointState.MarkStarted(); _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound"); } + /// + /// Releases an SDK whose StartAsync threw. The application instance may hold a half-opened + /// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the + /// transport is up); disposing it is the only way to be sure. The state stays Failed. + /// + private async Task ReleaseFailedStartAsync() + { + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is null) return; + try + { + await appHost.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw"); + } + } + + /// + /// Stops the SDK and disposes the application host, releasing endpoint ownership only when both + /// succeeded (): on a failure the + /// fields are retained so retries at host disposal. + /// + private async Task ReleaseEndpointAsync() + { + var server = _server; + var appHost = _appHost; + if (server is null && appHost is null) return; + var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( + async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); }, + () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask, + _logger).ConfigureAwait(false); + if (released) + { + _appHost = null; + _server = null; + } + } + /// /// The ServerHistorian:HonorReturnBounds kill switch StartAsync applies to the node manager /// (default true when the section or key is absent). Exposed so the wiring is testable without booting the SDK. @@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } /// - /// Stops the OPC UA server asynchronously. + /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes. /// + /// + /// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown, + /// the other hosted services' StopAsync), serving the last-written values with an emptied authority: + /// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was + /// up but inert instead of the connection-refused that triggers their failover. + /// stays idempotent with this: ownership is nulled once the stop succeeded. + /// /// Cancellation token. /// A task that represents the asynchronous operation. - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) + { + DetachSinks(); + await ReleaseEndpointAsync().ConfigureAwait(false); + if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped(); + } + + /// + /// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor + /// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running + /// until . + /// + private void DetachSinks() { // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose // authority and gate have already been cleared is the shape this whole teardown block exists to @@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // request rather than let the seams fall back to their role checks — a stopping node should shed // enforcement in the same order it acquired it. _server?.SetDataPlaneAuthorizer(null); - return Task.CompletedTask; } /// @@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl public async ValueTask DisposeAsync() { // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must - // not leave a tick armed against a server that is about to go away. + // not leave a tick armed against a server that is about to go away. A StopAsync that released the + // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left + // them set, and this is the retry. _revalidationTimer?.Dispose(); _revalidationTimer = null; - if (_appHost is not null) await _appHost.DisposeAsync(); + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is not null) await appHost.DisposeAsync(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 6404504e..d326d1b1 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -452,6 +452,11 @@ if (hasDriver) // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once. builder.Services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint + // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the + // interface, so the writer and the probe cannot disagree. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); } @@ -554,8 +559,9 @@ if (hasAdmin) builder.Services.AddOtOpcUaSecrets(builder.Configuration); // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so -// probing it would throw when the readiness endpoint resolves the factory. -builder.Services.AddOtOpcUaHealth(hasAdmin); +// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the +// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green. +builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver); builder.Services.AddOtOpcUaObservability(builder.Configuration); // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index 1a9730ce..d880e7b3 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -49,6 +49,34 @@ public sealed class ActiveTierRegistrationTests ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active); } + /// + /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not + /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + { + var registrations = new ServiceCollection() + .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .BuildServiceProvider() + .GetRequiredService>() + .Value.Registrations + .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) + .ToList(); + + if (!hasDriver) + { + registrations.ShouldBeEmpty(); + return; + } + + var registration = registrations.ShouldHaveSingleItem(); + registration.Tags.ShouldContain(ZbHealthTags.Ready); + registration.Tags.ShouldContain(ZbHealthTags.Active); + } + [Fact] public void Active_tier_check_is_not_on_the_ready_tier() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs new file mode 100644 index 00000000..341595d7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Health; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined +/// its mesh, and acked deploys through the Null sink while clients got connection-refused. The start +/// outcome is now a fact () the ready/active tiers read, a driver-only +/// node rethrows so the process exits, and StopAsync actually stops the SDK. +/// +public sealed class OtOpcUaServerHostedServiceStartFailureTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); + + /// + /// Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the + /// real /health/ready endpoint answers 503 — while StartAsync itself returns (fused node). + /// Positive control: the same endpoint answers 200 once the state is Started. + /// + [Fact] + public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null); + + var service = NewService(port, pki, state, fused); + try + { + await service.StartAsync(deadline.Token); // must NOT throw on a fused node + + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + state.FailureReason.ShouldNotBeNullOrWhiteSpace(); + + var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext()); + probe.Status.ShouldBe(HealthStatus.Unhealthy); + probe.Description.ShouldNotBeNull(); + probe.Description.ShouldContain("failed to start"); + + await using var web = BuildHealthHost(state); + await web.StartAsync(deadline.Token); + using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) }; + + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + + // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the + // 503 above measured the probe and not a broken health pipeline. + state.MarkStarted(); + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK); + await web.StopAsync(deadline.Token); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// A driver-only node has nothing else to stay up for: the failed start propagates. + [Fact] + public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null); + + var service = NewService(port, pki, state, driverOnly); + try + { + await Should.ThrowAsync(() => service.StartAsync(deadline.Token)); + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// + /// A clean start is Started and listening; StopAsync closes the port (connection refused, not + /// a stale-but-listening endpoint) and marks Stopped; DisposeAsync twice afterwards is a no-op. + /// + [Fact] + public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent() + { + using var deadline = new CancellationTokenSource(Deadline); + var port = FreePort(); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null)); + try + { + await service.StartAsync(deadline.Token); + state.Status.ShouldBe(OpcUaEndpointStatus.Started); + using (var client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token); + client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync"); + } + + await service.StopAsync(deadline.Token); + + state.Status.ShouldBe(OpcUaEndpointStatus.Stopped); + using (var client = new TcpClient()) + { + await Should.ThrowAsync(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token)); + } + + await service.DisposeAsync(); + await service.DisposeAsync(); // idempotent + } + finally + { + await service.DisposeAsync(); + Cleanup(pki); + } + } + + private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) => + new( + Options.Create(new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.StartFailure", + ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}", + OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki, + }), + new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(), + NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance, + EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(), + () => throw new NotSupportedException("no actor system required"), new ActorRegistry(), + null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance, + endpointState: state, roles: roles); + + /// The production health registration (driver role) over the supplied state, on a loopback Kestrel. + private static WebApplication BuildHealthHost(OpcUaEndpointState state) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(state); + builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true); + // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so + // the status measured is the endpoint's. (The registration shape itself is pinned separately.) + builder.Services.Configure(o => + { + foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList()) + o.Registrations.Remove(r); + }); + var app = builder.Build(); + app.MapOtOpcUaHealth(); + return app; + } + + /// + /// Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the + /// wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both + /// sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of + /// this test proved exactly that. The SDK also only fails when NEITHER family binds. + /// + private static TcpListener SquatAllInterfaces(out int port) + { + var squatter = new TcpListener(IPAddress.IPv6Any, 0); + squatter.Server.DualMode = true; + squatter.ExclusiveAddressUse = true; + squatter.Start(); + port = ((IPEndPoint)squatter.LocalEndpoint).Port; + return squatter; + } + + private static int FreePort() + { + using var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + return port; + } + + private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}"); + + private static void Cleanup(string pki) + { + if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true); + } +}