From 2d25c916b60578aab8f6a81fb76768aea8a8f3eb Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:30:14 -0400 Subject: [PATCH 1/8] fix(galaxy): make ShutdownAsync a real runtime teardown so re-init rebuilds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GalaxyDriver.ShutdownAsync was a no-op: the MX session, both owned gRPC channels, the EventPump, the alarm feed, the DeployWatcher, the probe watcher and the reconnect supervisor were released only by DisposeAsync, which the host never calls — one leaked gateway session per respawn, removal and /raw browse, and a watchdog Shutdown→Initialize that neither tore down nor rebuilt anything (review 2026-10-21 G-1, G-2, G-9, G-3). - ShutdownAsync and DisposeAsync share one ordered TeardownRuntimeAsync (supervisor first, then deploy/probe watchers, pump, alarm feed, session, channels); owned seams are nulled so the next InitializeAsync rebuilds; injected seams survive a shutdown and drop only on the final dispose. Lifecycle calls are serialised behind a gate. - A failed production connect releases its partial runtime (no orphaned client per watchdog retry) and lands Faulted before and after. - StartDeployWatcherAsync stops the previous watcher before starting a replacement, and now starts on a production instance's FIRST InitializeAsync (the old guard returned whenever discovery had not run here, which on a runtime instance is always). - ReconnectSupervisor releases each cycle's CancellationTokenSource when its loop exits and gains an awaitable StopAsync; PerPlatformProbeWatcher gains an awaitable StopAsync. - EventPump treats any exception under cancellation as a clean stop, so a teardown never wakes the supervisor. - Internal seams for tests: an IGalaxyDeployWatchSource ctor parameter and a session-open override forwarded to the owned GalaxyMxSession. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../GalaxyDriver.cs | 419 ++++++++++++++---- .../Health/PerPlatformProbeWatcher.cs | 20 +- .../Runtime/EventPump.cs | 6 +- .../Runtime/ReconnectSupervisor.cs | 58 ++- .../Runtime/SubscriptionRegistry.cs | 11 + .../Runtime/GalaxyDriverShutdownTests.cs | 375 ++++++++++++++++ .../ReconnectSupervisorCycleCtsTests.cs | 92 ++++ 7 files changed, 879 insertions(+), 102 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 3216e136..a75ba588 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -146,9 +146,61 @@ public sealed class GalaxyDriver private HostConnectivityForwarder? _transportForwarder; private PerPlatformProbeWatcher? _probeWatcher; + // Injected deploy-watch seam (tests). Null in production, where StartDeployWatcherAsync builds a + // GatewayGalaxyDeployWatchSource over the owned GalaxyRepositoryClient. + private readonly IGalaxyDeployWatchSource? _deployWatchSource; + + // True while the seams (_subscriber / _dataWriter / _hierarchySource / _alarmFeed / + // _alarmAcknowledger) were built by BuildProductionRuntimeAsync and are therefore ours to dispose + // and null on ShutdownAsync — the host's comms-loss recovery is ShutdownAsync → InitializeAsync on + // this SAME object, and InitializeAsync rebuilds only when the seams are null. Injected seams + // belong to the injector: ShutdownAsync stops every loop it started on top of them but leaves the + // seams in place, so a seam-driven test can cycle the lifecycle the way the watchdog does. + private bool _ownsRuntime; + + // Serialises InitializeAsync / ShutdownAsync / DisposeAsync so a late watchdog re-init can never + // interleave with a teardown and resurrect half of the runtime it is tearing down. + private readonly SemaphoreSlim _lifecycleGate = new(1, 1); + + // Set for the duration of a runtime teardown so the pump's stream-fault / replay-gap callbacks and + // the supervisor's replay cannot start a fresh pump underneath it. + private volatile bool _runtimeStopping; + private DriverHealth _health = new(DriverState.Unknown, null, null); private bool _disposed; + /// + /// Test seam — forwarded to on the + /// session creates, so the owned-runtime lifecycle + /// (client + session construction, teardown, rebuild) can be exercised without a gateway. The + /// gRPC channels it builds are real but lazy: nothing dials until a call is made. + /// + internal Func? SessionOpenOverrideForTests { get; set; } + + /// Test seam: the owned gateway client, null when no production runtime is live. + internal MxGatewayClient? OwnedMxClientForTests => _ownedMxClient; + + /// Test seam: the owned MX session, null when no production runtime is live. + internal GalaxyMxSession? OwnedMxSessionForTests => _ownedMxSession; + + /// Test seam: the owned repository client, null until the deploy watcher or discovery built it. + internal GalaxyRepositoryClient? OwnedRepositoryClientForTests => _ownedRepositoryClient; + + /// Test seam: the live reconnect supervisor, null outside a production runtime. + internal ReconnectSupervisor? SupervisorForTests => _supervisor; + + /// Test seam: the live per-platform probe watcher, null until InitializeAsync. + internal PerPlatformProbeWatcher? ProbeWatcherForTests => _probeWatcher; + + /// Test seam: the live deploy watcher, null until InitializeAsync starts one. + internal DeployWatcher? DeployWatcherForTests => _deployWatcher; + + /// Test seam: true while an is running. + internal bool HasEventPumpForTests { get { lock (_pumpLock) return _eventPump is not null; } } + + /// Test seam: the number of driver subscriptions the registry still tracks. + internal int TrackedSubscriptionCountForTests => _subscriptions.TrackedSubscriptionCount; + /// /// Server-pushed data-change notification. Fires from the /// 's background loop; handlers should be cheap (or queue @@ -202,6 +254,10 @@ public sealed class GalaxyDriver /// null-object resolver (returns null for every name) so seam-injecting tests that /// don't exercise a secret: ref need not supply one. /// + /// + /// Optional deploy-event stream seam. When supplied, starts the + /// over it (production builds one over the owned repository client). + /// internal GalaxyDriver( string driverInstanceId, GalaxyDriverOptions options, @@ -212,8 +268,10 @@ public sealed class GalaxyDriver IGalaxyAlarmAcknowledger? alarmAcknowledger = null, IGalaxyAlarmFeed? alarmFeed = null, ILogger? logger = null, - ISecretResolver? secretResolver = null) + ISecretResolver? secretResolver = null, + IGalaxyDeployWatchSource? deployWatchSource = null) { + _deployWatchSource = deployWatchSource; _driverInstanceId = !string.IsNullOrWhiteSpace(driverInstanceId) ? driverInstanceId : throw new ArgumentException("Driver instance id required.", nameof(driverInstanceId)); @@ -359,28 +417,47 @@ public sealed class GalaxyDriver ObjectDisposedException.ThrowIf(_disposed, this); if (HasConfigBody(driverConfigJson)) RefuseNonEquivalentConfig(driverConfigJson); - // Tests inject seams via the internal ctor; production InitializeAsync builds - // the gateway client + session + per-capability runtime components from - // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the - // test exercises the wired surface without a real gw round-trip). - if (_subscriber is null && _dataWriter is null && _hierarchySource is null) + await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false); - } - else - { - _logger.LogDebug( - "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped", - _driverInstanceId); - } + ObjectDisposedException.ThrowIf(_disposed, this); - await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false); - _logger.LogInformation( - "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", - _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName); - _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); + // Tests inject seams via the internal ctor; production InitializeAsync builds + // the gateway client + session + per-capability runtime components from + // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the + // test exercises the wired surface without a real gw round-trip). After a + // ShutdownAsync the production seams are null again, so the watchdog's + // Shutdown → Initialize genuinely rebuilds the runtime on this same object. + if (_subscriber is null && _dataWriter is null && _hierarchySource is null) + { + await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false); + } + else + { + _logger.LogDebug( + "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped", + _driverInstanceId); + // The per-platform probe watcher needs only a subscriber, so it runs on the seam path + // too — that is what lets the ScanState wiring be exercised without a gateway. + if (_subscriber is not null) _probeWatcher ??= BuildProbeWatcher(_subscriber); + } + + await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", + _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName); + _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); + } + finally + { + _lifecycleGate.Release(); + } } + private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) => + new(subscriber, _hostStatuses, _logger, + bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs); + /// /// Build the production gw client + session + per-capability runtime components /// from _options. Sets up the reconnect supervisor's reopen / replay @@ -389,9 +466,32 @@ public sealed class GalaxyDriver /// private async Task BuildProductionRuntimeAsync(CancellationToken cancellationToken) { + try + { + await BuildProductionRuntimeCoreAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + // A failed connect must not leave a half-built runtime behind: the watchdog retries + // InitializeAsync every 10 s, and the seam check above would see the orphaned client + + // session as "already built" — or, worse, rebuild on top of them. Release what was + // created, and land the failed state before AND after the teardown (IDriver remark). + _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message); + await TeardownRuntimeAsync(final: false, CancellationToken.None).ConfigureAwait(false); + _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message); + throw; + } + } + + private async Task BuildProductionRuntimeCoreAsync(CancellationToken cancellationToken) + { + _ownsRuntime = true; var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false); _ownedMxClient = MxGatewayClient.Create(clientOptions); - _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger); + _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger) + { + OpenAndRegisterOverrideForTests = SessionOpenOverrideForTests, + }; await _ownedMxSession.ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false); // PR 6.1 — wrap the gw-facing seams in tracing decorators so every Subscribe / @@ -424,9 +524,7 @@ public sealed class GalaxyDriver _supervisor.StateChanged += OnSupervisorStateChanged; - _probeWatcher = new PerPlatformProbeWatcher( - _subscriber, _hostStatuses, _logger, - bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs); + _probeWatcher = BuildProbeWatcher(_subscriber); // Wire the alarm acknowledger + feed to the live gateway client. Both are // session-less — the gateway serves alarms from an always-on central monitor — @@ -436,6 +534,133 @@ public sealed class GalaxyDriver _ownedMxClient.StreamAlarmsAsync, _logger, _options.MxAccess.ClientName); } + /// + /// Stop every background loop and release every gateway resource this driver holds — the + /// teardown half shared by ( = false: + /// a later rebuilds) and + /// ( = true). Before this existed, ShutdownAsync was a + /// no-op and everything below outlived the driver on every respawn, removal and + /// /raw browse: the gateway accumulated live MX sessions and advised items, and + /// this process accumulated channels and stream loops. + /// + /// + /// Ordering matters: + /// + /// The supervisor goes first and is awaited, so no recovery loop can reopen the + /// session or start a pump while the rest is being released. + /// The probe watcher unsubscribes while the session is still open (its handles + /// die with the session anyway, but a clean release keeps the gateway's advise + /// count honest). + /// Streams (pump, alarm feed, deploy watcher) are cancelled and awaited. + /// The session closes (gateway CloseSession), then both owned channels. + /// + /// Every step is best-effort: one failing release never prevents the next. + /// + private async Task TeardownRuntimeAsync(bool final, CancellationToken cancellationToken) + { + _runtimeStopping = true; + try + { + var supervisor = _supervisor; + _supervisor = null; + if (supervisor is not null) + { + supervisor.StateChanged -= OnSupervisorStateChanged; + try { await supervisor.StopAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor stop failed"); } + } + + var deployWatcher = _deployWatcher; + _deployWatcher = null; + if (deployWatcher is not null) + { + deployWatcher.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded; + try { await deployWatcher.StopAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher stop failed"); } + } + + var probeWatcher = _probeWatcher; + _probeWatcher = null; + if (probeWatcher is not null) + { + try { await probeWatcher.StopAsync(cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher stop failed"); } + } + + try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); } + _transportForwarder = null; + + EventPump? pump; + lock (_pumpLock) { pump = _eventPump; _eventPump = null; } + if (pump is not null) + { + pump.OnDataChange -= OnPumpDataChange; + try { await pump.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); } + } + // Every item handle the registry knows died with the pump's session; the host + // re-subscribes (fresh ids) once the next InitializeAsync reports Connected. + _subscriptions.Clear(); + + // The alarm feed: owned → disposed and nulled so InitializeAsync builds a fresh one; + // injected → detached (the injector owns its lifetime) unless this is the final dispose. + IGalaxyAlarmFeed? alarmFeed; + bool releaseFeed; + lock (_alarmFeedLock) + { + alarmFeed = _alarmFeed; + releaseFeed = alarmFeed is not null && (_ownsRuntime || final); + if (alarmFeed is not null && _alarmFeedWired) + { + alarmFeed.OnAlarmTransition -= OnAlarmFeedTransition; + _alarmFeedWired = false; + } + if (releaseFeed) _alarmFeed = null; + } + if (releaseFeed) + { + try { await alarmFeed!.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); } + } + + if (_ownedMxSession is not null) + { + try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); } + _ownedMxSession = null; + } + + if (_ownedMxClient is not null) + { + try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); } + _ownedMxClient = null; + } + + if (_ownedRepositoryClient is not null) + { + try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); } + _ownedRepositoryClient = null; + } + + if (_ownsRuntime || final) + { + // Null the seams so the next InitializeAsync rebuilds (production) — injected seams + // are only dropped on the final dispose. + _subscriber = null; + _dataWriter = null; + _hierarchySource = null; + _alarmAcknowledger = null; + _ownsRuntime = false; + } + } + finally + { + _runtimeStopping = false; + } + } + /// /// Reopen callback for : recreate the gw session. After a /// gateway restart the existing session handle is faulted / stale but non-null, so a plain @@ -465,6 +690,9 @@ public sealed class GalaxyDriver /// private async Task ReplayAsync(CancellationToken cancellationToken) { + // A teardown cancels the supervisor's cycle between reopen and replay; a replay that ran on + // anyway would start a fresh pump underneath the shutdown. + cancellationToken.ThrowIfCancellationRequested(); if (_subscriber is null) return; var entries = _subscriptions.SnapshotEntries(); if (entries.Count == 0) return; @@ -581,9 +809,9 @@ public sealed class GalaxyDriver } // EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the - // recreate if the driver is being disposed — Dispose already tore the pump down - // and a fresh one here would leak past the driver's lifetime. - if (_disposed) return; + // recreate if the driver is being disposed or its runtime torn down — the teardown + // already released the pump and a fresh one here would leak past it. + if (_disposed || _runtimeStopping) return; EnsureEventPumpStarted(); } @@ -631,28 +859,53 @@ public sealed class GalaxyDriver private async Task StartDeployWatcherAsync(CancellationToken cancellationToken) { if (!_options.Repository.WatchDeployEvents) return; - if (_ownedRepositoryClient is null && _hierarchySource is null) return; - // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand). - // If discovery hasn't run yet, build the client here so the watcher has a target. - // Build under a null-check (not ??=) so if BuildDefaultHierarchySourceAsync later - // runs it reuses this client rather than overwriting the field and leaking the - // first instance — the client-options build is now async (secret: arm). - _ownedRepositoryClient ??= ZB.MOM.WW.MxGateway.Client.GalaxyRepositoryClient.Create( - await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + var source = _deployWatchSource; + if (source is null) + { + // Seam mode without an injected watch source has no gateway to watch. (Production + // always owns its runtime here.) The old guard returned whenever BOTH the repository + // client and the hierarchy source were null — which on a production runtime instance + // is every first InitializeAsync, since discovery runs on the /raw picker's throwaway + // instance, not here — so the watcher never started and the re-browse chip never + // fired for a live driver. + if (!_ownsRuntime) return; - var source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient); - _deployWatcher = new DeployWatcher(source, _logger); - _deployWatcher.OnRediscoveryNeeded += (_, args) => OnRediscoveryNeeded?.Invoke(this, args); + // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand). + // Build under a null-check (not an overwrite) so if BuildDefaultHierarchySourceAsync + // later runs it reuses this client rather than leaking the first instance. + _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( + await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient); + } + + // One watcher per driver: a previous one (an InitializeAsync without a ShutdownAsync in + // between) is stopped before its replacement starts — every live InitializeAsync used to + // stack another watch-stream loop, each raising OnRediscoveryNeeded per real deploy. + var previous = _deployWatcher; + _deployWatcher = null; + if (previous is not null) + { + previous.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded; + try { await previous.StopAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "Previous DeployWatcher stop failed — replacing it anyway."); } + } + + var watcher = new DeployWatcher(source, _logger); + watcher.OnRediscoveryNeeded += OnDeployWatcherRediscoveryNeeded; + _deployWatcher = watcher; // StartAsync schedules the background loop and returns Task.CompletedTask immediately. // It throws InvalidOperationException synchronously if called twice (programming error). // Don't discard the return value — observe any synchronous throw. - var startTask = _deployWatcher.StartAsync(CancellationToken.None); + var startTask = watcher.StartAsync(CancellationToken.None); // The task is already completed (StartAsync is synchronous); surface any synchronous fault. if (startTask.IsFaulted) startTask.GetAwaiter().GetResult(); } + private void OnDeployWatcherRediscoveryNeeded(object? sender, RediscoveryEventArgs args) => + OnRediscoveryNeeded?.Invoke(this, args); + /// public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { @@ -731,13 +984,32 @@ public sealed class GalaxyDriver && a.Reconnect == b.Reconnect && a.ProbeTimeoutSeconds == b.ProbeTimeoutSeconds; + /// + /// Stop the driver and release the gateway runtime: supervisor, deploy watcher, probe + /// watcher, event pump, alarm feed, MX session (gateway CloseSession) and both owned + /// gRPC channels. The seams are nulled afterwards, so a later + /// on this same object — the host's comms-loss recovery and the way ReinitializeAsync + /// is sequenced — rebuilds a fresh runtime instead of reusing a dead one. The driver object + /// itself stays usable; is the final, idempotent teardown. + /// /// - public Task ShutdownAsync(CancellationToken cancellationToken) + public async Task ShutdownAsync(CancellationToken cancellationToken) { - if (_disposed) return Task.CompletedTask; + if (_disposed) return; _logger.LogInformation("GalaxyDriver {InstanceId} shutting down", _driverInstanceId); - _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); - return Task.CompletedTask; + await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_disposed) return; + await TeardownRuntimeAsync(final: false, cancellationToken).ConfigureAwait(false); + // Terminal health lands AFTER the teardown (IDriver remark): a loop unwinding during the + // release could otherwise overwrite it with the state it was reporting when cancelled. + _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); + } + finally + { + _lifecycleGate.Release(); + } } /// @@ -1192,7 +1464,7 @@ public sealed class GalaxyDriver /// private void OnEventPumpReplayGap(ReplayGap gap) { - if (_subscriber is null) return; + if (_subscriber is null || _runtimeStopping) return; if (!_options.Reconnect.ReplayOnSessionLost) { @@ -1249,6 +1521,14 @@ public sealed class GalaxyDriver /// private void OnEventPumpStreamFault(Exception cause) { + if (_runtimeStopping) + { + // The pump is being torn down with the rest of the runtime — a fault it reports on the + // way out is shutdown noise, not something to recover from. + _logger.LogDebug(cause, "GalaxyDriver {InstanceId} event stream ended during runtime teardown.", _driverInstanceId); + return; + } + var supervisor = _supervisor; if (supervisor is null) { @@ -1517,54 +1797,19 @@ public sealed class GalaxyDriver public async ValueTask DisposeAsync() { if (_disposed) return; - _disposed = true; - - // Synchronous sub-components first — none of these block. - try { _deployWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher dispose failed"); } - try { _supervisor?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor dispose failed"); } - try { _probeWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher dispose failed"); } - try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); } - - // Async sub-components: await each so we don't block a thread-pool thread - // on a slow shutdown (e.g. EventPump draining its channel, gRPC stream closing). - EventPump? pump; - lock (_pumpLock) { pump = _eventPump; _eventPump = null; } - if (pump is not null) + await _lifecycleGate.WaitAsync().ConfigureAwait(false); + try { - try { await pump.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); } + if (_disposed) return; + _disposed = true; + // Same release sequence as ShutdownAsync, plus the injected seams this instance will + // never need again. Idempotent: a ShutdownAsync before this finds nothing left to do. + await TeardownRuntimeAsync(final: true, CancellationToken.None).ConfigureAwait(false); } - - IGalaxyAlarmFeed? alarmFeed; - lock (_alarmFeedLock) { alarmFeed = _alarmFeed; _alarmFeed = null; } - if (alarmFeed is not null) + finally { - try { await alarmFeed.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); } + _lifecycleGate.Release(); } - - if (_ownedMxSession is not null) - { - try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); } - _ownedMxSession = null; - } - - if (_ownedMxClient is not null) - { - try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); } - _ownedMxClient = null; - } - - if (_ownedRepositoryClient is not null) - { - try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); } - _ownedRepositoryClient = null; - } - - _hierarchySource = null; } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs index b766e38b..12afb837 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs @@ -196,20 +196,28 @@ public sealed class PerPlatformProbeWatcher : IDisposable }; } - /// Disposes the probe watcher and unsubscribes all tracked platforms. - public void Dispose() + /// + /// Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway + /// (best-effort — a transport error is logged and swallowed). The async counterpart of + /// ; the driver's ShutdownAsync uses it so the unsubscribe + /// round-trip is awaited rather than blocked on. Idempotent. + /// + /// Bounds the unsubscribe round-trip. + /// A task that completes once the handles have been released (or the attempt gave up). + public async Task StopAsync(CancellationToken cancellationToken) { if (_disposed) return; _disposed = true; - // Best-effort unsubscribe everything we know about. Run synchronously through - // GetAwaiter().GetResult() since Dispose is sync; transport errors are swallowed. var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray(); _itemHandlesByPlatform.Clear(); if (liveHandles.Length > 0) { - try { _subscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None).GetAwaiter().GetResult(); } - catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher dispose unsubscribe failed"); } + try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); } } } + + /// Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip). + public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult(); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index d6e4fb4b..e39385b4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -187,9 +187,11 @@ internal sealed class EventPump : IAsyncDisposable NotifyStreamFault(cause); } } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + catch (Exception) when (ct.IsCancellationRequested) { - // Clean shutdown — no log. + // Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled) + // rather than OperationCanceledException; either way the pump was told to stop, so this + // is not a transport fault and must not wake the reconnect supervisor mid-teardown. } catch (Exception ex) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs index 15022173..7b6c6410 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs @@ -62,6 +62,16 @@ public sealed class ReconnectSupervisor : IDisposable private Task? _recoveryLoop; private CancellationTokenSource? _loopCts; private bool _disposed; + private int _cyclesStarted; + private int _cyclesReleased; + + /// Recovery cycles started so far (one per that left Healthy). Test seam. + internal int CyclesStarted => Volatile.Read(ref _cyclesStarted); + + /// Recovery cycles whose has been disposed. Equals + /// once every loop has exited — a cycle's CTS used to be released only + /// if it happened to be the LAST one when ran. Test seam. + internal int CyclesReleased => Volatile.Read(ref _cyclesReleased); /// Fires after every state transition. public event EventHandler? StateChanged; @@ -136,8 +146,13 @@ public sealed class ReconnectSupervisor : IDisposable TransitionLocked(State.TransportLost, cause.Message); - _loopCts = new CancellationTokenSource(); - _recoveryLoop = Task.Run(() => RecoveryLoopAsync(_loopCts.Token)); + // Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's + // finally). Before, every cycle allocated a fresh CTS and only the last one was ever + // disposed — one leaked CancellationTokenSource per outage for the driver's lifetime. + var cts = new CancellationTokenSource(); + _loopCts = cts; + Interlocked.Increment(ref _cyclesStarted); + _recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts)); } } @@ -156,7 +171,26 @@ public sealed class ReconnectSupervisor : IDisposable } } - private async Task RecoveryLoopAsync(CancellationToken ct) + private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts) + { + try + { + await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false); + } + finally + { + // Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may + // race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object. + lock (_stateLock) + { + if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null; + } + cycleCts.Dispose(); + Interlocked.Increment(ref _cyclesReleased); + } + } + + private async Task RecoveryLoopCoreAsync(CancellationToken ct) { var attempt = 0; while (!ct.IsCancellationRequested) @@ -240,8 +274,13 @@ public sealed class ReconnectSupervisor : IDisposable } } - /// Disposes the supervisor and cancels any in-flight recovery loop. - public void Dispose() + /// + /// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's + /// thread. The async counterpart of ; the driver's ShutdownAsync + /// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent. + /// + /// A task that completes once the recovery loop has exited. + public async Task StopAsync() { if (_disposed) return; _disposed = true; @@ -250,13 +289,18 @@ public sealed class ReconnectSupervisor : IDisposable Task? loop; lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; } - cts?.Cancel(); + // The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS + // throws on Cancel, and a disposed one is a no-op on Dispose. + try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ } if (loop is not null) { - try { loop.GetAwaiter().GetResult(); } catch { /* shutdown */ } + try { await loop.ConfigureAwait(false); } catch { /* shutdown */ } } cts?.Dispose(); } + + /// Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits). + public void Dispose() => StopAsync().GetAwaiter().GetResult(); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs index 512099f5..9dbd7755 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs @@ -100,6 +100,17 @@ internal sealed class SubscriptionRegistry } + /// + /// Drop every subscription and the whole reverse map. Used by the driver's runtime teardown: + /// once the gateway session is closed every item handle in here is dead, and the host + /// re-issues its subscriptions (fresh ids) after the next InitializeAsync. + /// + public void Clear() + { + _bySubscriptionId.Clear(); + _subscribersByItemHandle.Clear(); + } + /// Snapshot every active binding for diagnostic output. /// A flattened list of every tag binding across all tracked subscriptions. public IReadOnlyList SnapshotAllBindings() => diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs new file mode 100644 index 00000000..5a146656 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs @@ -0,0 +1,375 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// used to be a no-op (review 2026-10-21, G-1/G-2/G-9): +/// the gRPC runtime outlived the driver on every respawn, removal and /raw browse, and +/// the host's comms-loss recovery (ShutdownAsync → InitializeAsync on the SAME +/// object) neither tore down nor rebuilt anything. These tests pin the teardown at every +/// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed, +/// the owned channels + session — and the rebuild on the next InitializeAsync. +/// +public sealed class GalaxyDriverShutdownTests +{ + private const int WaitMs = 10_000; + + private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("ShutdownTest"), + new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), + new GalaxyReconnectOptions()); + + // ───────────────────────────── fakes ───────────────────────────── + + /// + /// Subscriber whose event stream records the token it was opened with and completes a + /// signal when that token fires — the observable for "the StreamEvents stream was cancelled". + /// + private sealed class StreamingSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + private int _nextHandle; + + public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); + public List Unsubscribed { get; } = []; + public int StreamsOpened; + public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var results = new List(); + foreach (var r in fullReferences) + { + var h = Interlocked.Increment(ref _nextHandle); + HandleByRef[r] = h; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + } + return Task.FromResult>(results); + } + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) + { + lock (Unsubscribed) Unsubscribed.AddRange(itemHandles); + return Task.CompletedTask; + } + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref StreamsOpened); + using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult()); + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { DoubleValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + /// + /// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when + /// the enumerator is cancelled or completes) — the observable for "exactly one watcher". + /// + private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource + { + private int _open; + private int _opened; + public Channel Current { get; private set; } = Channel.CreateUnbounded(); + + public int Open => Volatile.Read(ref _open); + public int Opened => Volatile.Read(ref _opened); + + public async IAsyncEnumerable WatchAsync( + DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref _open); + Interlocked.Increment(ref _opened); + var channel = Current = Channel.CreateUnbounded(); + try + { + await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + finally + { + Interlocked.Decrement(ref _open); + } + } + + public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent + { + Sequence = sequence, + TimeOfLastDeployPresent = true, + TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime), + }); + } + + private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed + { + public int Starts; + public int Disposes; + public event EventHandler? OnAlarmTransition; + public void Start() => Starts++; + public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition( + "Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise, + AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi")); + public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; } + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } + + // ───────────────────────────── G-1: streams + pump ───────────────────────────── + + /// + /// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the + /// registry is empty. Positive control: an event delivered through that same stream + /// reached OnDataChange before the shutdown. + /// + [Fact] + public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump() + { + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + driver.OnDataChange += (_, e) => delivered.TrySetResult(e); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeTrue(); + + await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0); + (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task, + "positive control: the stream must deliver before the shutdown"); + + await driver.ShutdownAsync(CancellationToken.None); + + (await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task, + "ShutdownAsync must cancel the StreamEvents stream"); + driver.HasEventPumpForTests.ShouldBeFalse(); + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init"); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown"); + } + + /// + /// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT + /// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive + /// control on both sides of the shutdown. + /// + [Fact] + public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it() + { + var feed = new RecordingAlarmFeed(); + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, + subscriber: sub, alarmFeed: feed); + var received = 0; + driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received); + + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None); + feed.Emit(); + received.ShouldBe(1, "positive control: the feed is wired before the shutdown"); + + await driver.ShutdownAsync(CancellationToken.None); + feed.Emit(); + received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent"); + feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector"); + + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None); + feed.Emit(); + received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed"); + feed.Starts.ShouldBe(2); + } + + // ───────────────────────────── G-2: one DeployWatcher ───────────────────────────── + + /// + /// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch + /// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly + /// once — five stacked watchers would raise five times. + /// + [Fact] + public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher() + { + var source = new CountingDeployWatchSource(); + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + deployWatchSource: source); + var raised = 0; + driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); + + await driver.InitializeAsync("{}", CancellationToken.None); + for (var cycle = 0; cycle < 5; cycle++) + { + // Let this cycle's watch stream actually open before the next shutdown, so every cycle + // holds a live stream the teardown has to cancel (a loop cancelled before its first + // open would make the count vacuous). + await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open"); + await driver.ShutdownAsync(CancellationToken.None); + await driver.InitializeAsync("{}", CancellationToken.None); + } + + await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1, + $"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}"); + driver.DeployWatcherForTests.ShouldNotBeNull(); + + // Bootstrap (suppressed) then a genuine change. + await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); + await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery"); + await Task.Delay(200); + Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy"); + + await driver.ShutdownAsync(CancellationToken.None); + await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream"); + driver.DeployWatcherForTests.ShouldBeNull(); + } + + /// A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one. + [Fact] + public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher() + { + var source = new CountingDeployWatchSource(); + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + deployWatchSource: source); + + await driver.InitializeAsync("{}", CancellationToken.None); + var first = driver.DeployWatcherForTests; + await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open"); + await driver.InitializeAsync("{}", CancellationToken.None); + + await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1, + $"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}"); + driver.DeployWatcherForTests.ShouldNotBeSameAs(first); + } + + // ───────────────────────────── G-1/G-9: owned runtime, channels + session ───────────────────────────── + + /// + /// The production path (no injected seams): InitializeAsync builds the owned client, session + /// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the + /// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's + /// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC + /// channels are real but lazy (nothing dials until a call is made). + /// + [Fact] + public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds() + { + var opens = 0; + var source = new CountingDeployWatchSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source) + { + SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; }, + }; + + await driver.InitializeAsync("{}", CancellationToken.None); + opens.ShouldBe(1); + var client = driver.OwnedMxClientForTests.ShouldNotBeNull(); + var session = driver.OwnedMxSessionForTests.ShouldNotBeNull(); + session.IsConnected.ShouldBeTrue(); + driver.SupervisorForTests.ShouldNotBeNull(); + driver.ProbeWatcherForTests.ShouldNotBeNull(); + driver.DeployWatcherForTests.ShouldNotBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + + await driver.ShutdownAsync(CancellationToken.None); + + session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session"); + await Should.ThrowAsync(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None), + "the owned gateway channel must be disposed"); + driver.OwnedMxClientForTests.ShouldBeNull(); + driver.OwnedMxSessionForTests.ShouldBeNull(); + driver.SupervisorForTests.ShouldBeNull(); + driver.ProbeWatcherForTests.ShouldBeNull(); + driver.DeployWatcherForTests.ShouldBeNull(); + await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled"); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp); + + // The watchdog's second half: a fresh runtime on the same object. + await driver.InitializeAsync("{}", CancellationToken.None); + opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one"); + driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client); + driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session); + driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue(); + driver.SupervisorForTests.ShouldNotBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher"); + } + + /// + /// A connect that fails mid-build releases the partial runtime (no orphaned client per + /// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing. + /// + [Fact] + public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds() + { + var attempts = 0; + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null) + { + SessionOpenOverrideForTests = _ => ++attempts == 1 + ? throw new InvalidOperationException("gateway refused") + : Task.CompletedTask, + }; + + var ex = await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None)); + ex.Message.ShouldBe("gateway refused"); + driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client"); + driver.OwnedMxSessionForTests.ShouldBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Faulted); + driver.GetHealth().LastError.ShouldBe("gateway refused"); + + await driver.InitializeAsync("{}", CancellationToken.None); + attempts.ShouldBe(2); + driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue(); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op. + [Fact] + public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless() + { + var sub = new StreamingSubscriber(); + var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + + await driver.ShutdownAsync(CancellationToken.None); + await Should.NotThrowAsync(async () => await driver.DisposeAsync()); + await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None)); + await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None)); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs new file mode 100644 index 00000000..1660751d --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs @@ -0,0 +1,92 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-3: allocated a +/// new per recovery cycle and only the last one was ever +/// disposed. Each cycle now releases its own CTS when its loop exits, and +/// awaits the loop instead of blocking on it. +/// +public sealed class ReconnectSupervisorCycleCtsTests +{ + private const int WaitMs = 10_000; + + private static ReconnectOptions FastBackoff() => + new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20)); + + /// Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control). + [Fact] + public async Task Every_completed_recovery_cycle_releases_its_own_cts() + { + var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = false; + using var sup = new ReconnectSupervisor( + reopen: async _ => + { + if (hold) + { + reopenEntered.TrySetResult(); + await holdReopen.Task; + } + }, + replay: _ => Task.CompletedTask, + options: FastBackoff()); + + for (var i = 0; i < 3; i++) + { + sup.ReportTransportFailure(new IOException($"drop {i}")); + await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token); + } + await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS"); + sup.CyclesStarted.ShouldBe(3); + + // Positive control: a cycle parked inside reopen still holds its CTS. + hold = true; + sup.ReportTransportFailure(new IOException("drop 3")); + (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task); + sup.CyclesStarted.ShouldBe(4); + sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits"); + + holdReopen.TrySetResult(); + await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token); + await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit"); + } + + /// StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking. + [Fact] + public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts() + { + var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sup = new ReconnectSupervisor( + reopen: async ct => + { + reopenEntered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }, + replay: _ => Task.CompletedTask, + options: FastBackoff()); + + sup.ReportTransportFailure(new IOException("drop")); + (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task); + + var stop = sup.StopAsync(); + (await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return"); + sup.CyclesReleased.ShouldBe(1); + await Should.NotThrowAsync(sup.StopAsync); // idempotent + Should.NotThrow(sup.Dispose); + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } +} From 1b3169273d6e92a6d4dd81801a55a0d1c4307efe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:35:48 -0400 Subject: [PATCH 2/8] fix(galaxy): wire ScanState probing, single-sweep discovery, liveness stamp, dispatch-loop fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review 2026-10-21 G-4, G-7, G-11, G-10. - G-4: PerPlatformProbeWatcher publishes its probe handles into the driver's SubscriptionRegistry under the reserved ProbeSubscriptionId (-1), so the EventPump's fan-out finally reaches the watcher — every ScanState event used to be dropped as "stale after unsubscribe" and per-platform host status never updated. ResubscribeAllAsync delegates the reserved entry to the watcher's new ResubscribeAsync, so the probe survives a reconnect; StopAsync removes the entry. Probe traffic is routed by the reserved handle and never reaches the public OnDataChange. The watcher is also built on the seam path (subscriber only), which is what makes the wiring testable without a gateway. - G-7: GalaxyDiscoverer.DiscoverAsync returns the hierarchy it fetched and the driver selects its $WinPlatform/$AppEngine probe targets from that same sweep instead of asking the gateway for the whole galaxy twice. - G-11: OnPumpDataChange stamps LastSuccessfulRead from inbound stream data, throttled (1 s default), so the comms-loss watchdog has honest liveness evidence under a Degraded supervisor. - G-10: an EventPump dispatch-loop exit that is not a cancellation now reports a stream fault so the supervisor recreates the pump, instead of leaving a deaf pump reporting Healthy. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Browse/GalaxyDiscoverer.cs | 10 +- .../GalaxyDriver.cs | 66 ++++-- .../Health/PerPlatformProbeWatcher.cs | 117 ++++++++-- .../Runtime/EventPump.cs | 9 +- .../Runtime/SubscriptionRegistry.cs | 29 +++ .../PerPlatformProbeWatcherRegistryTests.cs | 112 +++++++++ .../Runtime/EventPumpDispatchFaultTests.cs | 96 ++++++++ .../Runtime/GalaxyDriverProbeWiringTests.cs | 217 ++++++++++++++++++ 8 files changed, 614 insertions(+), 42 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs index 6242351c..6e3bc5f8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs @@ -61,8 +61,12 @@ public sealed class GalaxyDiscoverer /// /// The address space builder to populate with discovery results. /// The cancellation token for the operation. - /// A task that represents the asynchronous operation. - public async Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken) + /// + /// The hierarchy this pass fetched, so a caller that needs the object set for something + /// else (the driver selects its $WinPlatform / $AppEngine probe targets from + /// it) reuses the one sweep instead of asking the gateway for the full galaxy twice. + /// + public async Task> DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(builder); var objects = await _source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false); @@ -132,6 +136,8 @@ public sealed class GalaxyDiscoverer } } } + + return objects; } // Resolve (and memoise) the folder for a gobject, recursively creating its parent's folder diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index a75ba588..f2768291 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -454,8 +454,11 @@ public sealed class GalaxyDriver } } + // The watcher publishes its probe handles into the driver's registry (reserved id) so the + // EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every + // probe event was dropped before it got anywhere near the watcher. private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) => - new(subscriber, _hostStatuses, _logger, + new(subscriber, _hostStatuses, _subscriptions, _logger, bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs); /// @@ -738,6 +741,15 @@ public sealed class GalaxyDriver var tagCount = 0; foreach (var (subscriptionId, oldBindings) in entries) { + if (subscriptionId == SubscriptionRegistry.ProbeSubscriptionId && _probeWatcher is { } probeWatcher) + { + // The probe watcher owns its handles: it re-subscribes its platforms, records the + // fresh handles and rebinds the reserved registry entry itself, so ScanState host + // status keeps updating after a reconnect (it used to go permanently deaf). + tagCount += await probeWatcher.ResubscribeAsync(cancellationToken).ConfigureAwait(false); + continue; + } + var refs = oldBindings .Select(b => b.FullReference) .Distinct(StringComparer.OrdinalIgnoreCase) @@ -1058,11 +1070,12 @@ public sealed class GalaxyDriver // keyed by RawPath, matching what WriteAsync resolves against. var discoverer = new GalaxyDiscoverer( source, attributeRef => _rawTagByAttributeRef.GetValueOrDefault(attributeRef)); - await discoverer.DiscoverAsync(capturingBuilder, cancellationToken).ConfigureAwait(false); + // One sweep: the discoverer hands back the hierarchy it materialised, and the platform + // selection below reads that instead of asking the gateway for the whole galaxy again. + var hierarchy = await discoverer.DiscoverAsync(capturingBuilder, cancellationToken).ConfigureAwait(false); if (_probeWatcher is not null) { - var hierarchy = await source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false); var platforms = hierarchy .Where(o => o.TemplateChain.Any(t => string.Equals(t, "$WinPlatform", StringComparison.OrdinalIgnoreCase) @@ -1744,6 +1757,24 @@ public sealed class GalaxyDriver /// private void OnPumpDataChange(object? sender, DataChangeEventArgs args) { + // Anything the gateway pushes on the stream proves it is alive and serving this session — + // the comms-loss watchdog's evidence (G-11: the stamp used to move only at init and on a + // supervisor Healthy transition, so a 90 s Degraded tripped a re-init under flowing data). + StampLiveness(); + + // Probe traffic rides the reserved registry entry and stays in the attributeRef domain — + // ScanState attributes are discovered platform attributes, never authored RawPaths. It is + // the watcher's alone: the host never subscribed it and must not see it on OnDataChange. + if (args.SubscriptionHandle is GalaxySubscriptionHandle { SubscriptionId: SubscriptionRegistry.ProbeSubscriptionId }) + { + // The probe decoder takes a raw quality byte. Recover it via the canonical + // StatusCodeMap.ToQualityCategoryByte helper so the mapping lives in one + // place next to its inverse (FromQualityByte) and cannot desync silently. + var qualityByte = StatusCodeMap.ToQualityCategoryByte(args.Snapshot.StatusCode); + _probeWatcher?.OnProbeValueChanged(args.FullReference, args.Snapshot.Value, qualityByte); + return; + } + // v3: the pump fans out keyed by the dialled attributeRef; the server correlates values to nodes // by RawPath (the node's FullName). Reverse-map before surfacing to public OnDataChange consumers. // Identity on a miss, so a live-browse tag surfaces under its attributeRef unchanged. @@ -1754,18 +1785,25 @@ public sealed class GalaxyDriver publicArgs = args with { FullReference = rawPath }; } OnDataChange?.Invoke(this, publicArgs); + } - // Probe watching stays in the attributeRef domain — ScanState attributes are discovered platform - // attributes, never authored RawPaths, so use the untranslated reference here. - if (_probeWatcher is not null - && args.FullReference.EndsWith(PerPlatformProbeWatcher.ProbeSuffix, StringComparison.OrdinalIgnoreCase)) - { - // The probe decoder takes a raw quality byte. Recover it via the canonical - // StatusCodeMap.ToQualityCategoryByte helper so the mapping lives in one - // place next to its inverse (FromQualityByte) and cannot desync silently. - var qualityByte = StatusCodeMap.ToQualityCategoryByte(args.Snapshot.StatusCode); - _probeWatcher.OnProbeValueChanged(args.FullReference, args.Snapshot.Value, qualityByte); - } + /// Minimum gap between two LastSuccessfulRead stamps from stream data. Internal so tests can widen it. + internal TimeSpan LivenessStampThrottle { get; set; } = TimeSpan.FromSeconds(1); + + private long _lastLivenessStampTicks; + + /// + /// Advance LastSuccessfulRead from inbound stream data, at most once per + /// — this runs on the pump's dispatch loop for every + /// event, so a 50k-tag galaxy must not pay a health-record allocation per value. + /// + private void StampLiveness() + { + var now = DateTime.UtcNow; + var last = Volatile.Read(ref _lastLivenessStampTicks); + if (now.Ticks - last < LivenessStampThrottle.Ticks) return; + Volatile.Write(ref _lastLivenessStampTicks, now.Ticks); + _health = _health with { LastSuccessfulRead = now }; } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs index 12afb837..ae4c41ac 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs @@ -38,6 +38,12 @@ public sealed class PerPlatformProbeWatcher : IDisposable private readonly ILogger _logger; private readonly int _bufferedUpdateIntervalMs; + // The driver's fan-out registry. The EventPump resolves every inbound event through it, so + // the probe handles must be registered there (under SubscriptionRegistry.ProbeSubscriptionId) + // or every ScanState event is dropped as "stale after unsubscribe". Null when the watcher is + // driven directly (unit tests of the decoding table). + private readonly SubscriptionRegistry? _registry; + // Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe; // we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call. private readonly ConcurrentDictionary _itemHandlesByPlatform = @@ -55,9 +61,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable HostStatusAggregator aggregator, ILogger? logger = null, int bufferedUpdateIntervalMs = 0) + : this(subscriber, aggregator, registry: null, logger, bufferedUpdateIntervalMs) + { + } + + /// + /// Driver-side ctor: the watcher publishes its probe handles into + /// under so the + /// fans ScanState events out to . + /// + /// The Galaxy subscriber for managing probe subscriptions. + /// The host status aggregator for tracking platform connectivity. + /// The driver's subscription registry; null to keep the handles private. + /// Optional logger for diagnostic messages. + /// Buffered update interval in milliseconds; must be >= 0. + internal PerPlatformProbeWatcher( + IGalaxySubscriber subscriber, + HostStatusAggregator aggregator, + SubscriptionRegistry? registry, + ILogger? logger = null, + int bufferedUpdateIntervalMs = 0) { _subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber)); _aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator)); + _registry = registry; _logger = logger ?? NullLogger.Instance; if (bufferedUpdateIntervalMs < 0) { @@ -67,6 +94,68 @@ public sealed class PerPlatformProbeWatcher : IDisposable _bufferedUpdateIntervalMs = bufferedUpdateIntervalMs; } + /// Current platform → probe-handle bindings, as the registry sees them. Test seam. + internal IReadOnlyList CurrentBindings => BuildBindings(); + + private IReadOnlyList BuildBindings() => + [.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))]; + + /// Mirror the tracked handle set into the driver registry (no-op without one). + private void PublishBindings() + { + if (_registry is null) return; + var bindings = BuildBindings(); + if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId); + else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings); + } + + /// + /// Re-issue SubscribeBulk for every tracked platform and replace the handles — the + /// reconnect replay path, where the gateway session was recreated and every pre-reconnect + /// handle is dead. Platforms the gateway now rejects keep a 0 handle (no events expected). + /// + /// A cancellation token to cancel the operation. + /// The number of probe references re-subscribed. + public async Task ResubscribeAsync(CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + var platforms = _itemHandlesByPlatform.Keys.ToArray(); + if (platforms.Length == 0) return 0; + + await SubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false); + return platforms.Length; + } + + private async Task SubscribeAndRecordAsync(IReadOnlyList platforms, CancellationToken cancellationToken) + { + var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray(); + // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence + // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState + // changes are rare so a coarser interval is usually fine; deployments that need + // tighter health visibility can dial it down through GalaxyDriverOptions. + var results = await _subscriber.SubscribeBulkAsync( + probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false); + + for (var i = 0; i < platforms.Count; i++) + { + var platform = platforms[i]; + var match = results.FirstOrDefault(r => string.Equals( + r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase)); + + var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0; + _itemHandlesByPlatform[platform] = itemHandle; + + if (itemHandle <= 0) + { + _logger.LogWarning( + "PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", + platform, match?.ErrorMessage ?? ""); + } + } + + PublishBindings(); + } + /// Snapshot of platform tag names currently watched. public IReadOnlyCollection WatchedPlatforms => [.. _itemHandlesByPlatform.Keys]; @@ -109,6 +198,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable if (toRemove.Count > 0) { + // Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe. + PublishBindings(); try { await _subscriber.UnsubscribeBulkAsync( @@ -124,30 +215,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable if (toAdd.Count == 0) return; - var probeAddresses = toAdd.Select(p => p + ProbeSuffix).ToArray(); - // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence - // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState - // changes are rare so a coarser interval is usually fine; deployments that need - // tighter health visibility can dial it down through GalaxyDriverOptions. - var results = await _subscriber.SubscribeBulkAsync( - probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false); - - for (var i = 0; i < toAdd.Count; i++) - { - var platform = toAdd[i]; - var match = results.FirstOrDefault(r => string.Equals( - r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase)); - - var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0; - _itemHandlesByPlatform[platform] = itemHandle; - - if (itemHandle <= 0) - { - _logger.LogWarning( - "PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", - platform, match?.ErrorMessage ?? ""); - } - } + await SubscribeAndRecordAsync(toAdd, cancellationToken).ConfigureAwait(false); } /// @@ -211,6 +279,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray(); _itemHandlesByPlatform.Clear(); + _registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId); if (liveHandles.Length > 0) { try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index e39385b4..7eb95234 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -285,14 +285,19 @@ internal sealed class EventPump : IAsyncDisposable EventsDispatched.Add(1, _clientTag); } } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + catch (Exception) when (ct.IsCancellationRequested) { // Clean shutdown. } catch (Exception ex) { + // The read loop is still draining the stream into a channel nobody reads: the pump is + // as dead as after a transport fault, but used to raise nothing — health stayed + // Healthy and the comms-loss watchdog never tripped (a clean termination evading the + // watchdog). Report it as a stream fault so the supervisor recreates the pump. _logger.LogWarning(ex, - "Galaxy EventPump dispatch loop ended with an exception — events past this point will be lost until restart."); + "Galaxy EventPump dispatch loop ended with an exception — notifying reconnect supervisor so the pump is recreated."); + NotifyStreamFault(ex); } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs index 9dbd7755..1a6ac2ff 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs @@ -18,10 +18,39 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; /// internal sealed class SubscriptionRegistry { + /// + /// Reserved subscription id for the per-platform ScanState probe handles + /// (PerPlatformProbeWatcher). Driver subscription ids are positive and + /// monotonic (), so a negative id can never collide. + /// The probe handles used to live only in the watcher's own map, so the + /// — which resolves subscribers through THIS registry — + /// dropped every probe event as "stale after unsubscribe" and per-platform host + /// status never updated. + /// + public const long ProbeSubscriptionId = -1; + private readonly ConcurrentDictionary _bySubscriptionId = new(); private readonly ConcurrentDictionary> _subscribersByItemHandle = new(); private long _nextSubscriptionId; + /// True when is currently registered. + /// The subscription identifier. + /// Whether the id is tracked. + public bool Contains(long subscriptionId) => _bySubscriptionId.ContainsKey(subscriptionId); + + /// + /// when the id is already tracked (so its stale reverse-map entries + /// are dropped), otherwise. Used by the probe watcher, whose single + /// reserved subscription is re-issued on every platform-set change. + /// + /// The subscription identifier. + /// The tag bindings for the subscription. + public void Upsert(long subscriptionId, IReadOnlyList bindings) + { + if (Contains(subscriptionId)) Rebind(subscriptionId, bindings); + else Register(subscriptionId, bindings); + } + /// Gets the number of tracked subscriptions. public int TrackedSubscriptionCount => _bySubscriptionId.Count; /// Gets the number of tracked item handles. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs new file mode 100644 index 00000000..511bb696 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs @@ -0,0 +1,112 @@ +using System.Runtime.CompilerServices; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Health; + +/// +/// Review 2026-10-21 G-4: the watcher's probe handles must live in the driver's +/// under the reserved id, follow the platform set, be +/// rebound on , and leave on stop. +/// +public sealed class PerPlatformProbeWatcherRegistryTests +{ + private sealed class FakeSubscriber : IGalaxySubscriber + { + private int _next; + public List Unsubscribes { get; } = []; + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + => Task.FromResult>( + [.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = ++_next, WasSuccessful = true })]); + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) + { + Unsubscribes.Add([.. itemHandles]); + return Task.CompletedTask; + } + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + yield break; + } + } + + private static (long Id, string Ref)[] Resolve(SubscriptionRegistry registry, int handle) => + [.. registry.ResolveSubscribers(handle)]; + + /// Synced platforms resolve through the registry under the reserved id; removed ones stop resolving. + [Fact] + public async Task Sync_registers_probe_handles_under_the_reserved_id_and_tracks_removals() + { + var registry = new SubscriptionRegistry(); + using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry); + + await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None); + + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue(); + var bindings = watcher.CurrentBindings; + bindings.Count.ShouldBe(2); + foreach (var b in bindings) + { + Resolve(registry, b.ItemHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, b.FullReference)]); + } + + var bHandle = bindings.Single(b => b.FullReference == "B.ScanState").ItemHandle; + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + Resolve(registry, bHandle).ShouldBeEmpty("a removed platform's handle must leave the fan-out map"); + var aHandle = bindings.Single(b => b.FullReference == "A.ScanState").ItemHandle; + Resolve(registry, aHandle).ShouldHaveSingleItem("positive control: the kept platform still resolves"); + } + + /// Resubscribe replaces the handles and rebinds: old handles stop resolving, new ones resolve. + [Fact] + public async Task Resubscribe_rebinds_the_registry_to_the_fresh_handles() + { + var registry = new SubscriptionRegistry(); + using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry); + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + var oldHandle = watcher.CurrentBindings.Single().ItemHandle; + + (await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1); + + var newHandle = watcher.CurrentBindings.Single().ItemHandle; + newHandle.ShouldNotBe(oldHandle); + Resolve(registry, oldHandle).ShouldBeEmpty(); + Resolve(registry, newHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, "A.ScanState")]); + registry.TrackedSubscriptionCount.ShouldBe(1); + } + + /// StopAsync unsubscribes the live handles and removes the reserved entry. + [Fact] + public async Task StopAsync_unsubscribes_and_removes_the_reserved_entry() + { + var registry = new SubscriptionRegistry(); + var subscriber = new FakeSubscriber(); + var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry); + await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None); + var handles = watcher.CurrentBindings.Select(b => b.ItemHandle).OrderBy(h => h).ToArray(); + + await watcher.StopAsync(CancellationToken.None); + + subscriber.Unsubscribes.ShouldHaveSingleItem().OrderBy(h => h).ShouldBe(handles); + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse(); + registry.TrackedItemHandleCount.ShouldBe(0); + await Should.NotThrowAsync(() => watcher.StopAsync(CancellationToken.None)); // idempotent + } + + /// A watcher without a registry (the decoding-table unit tests) still works and registers nothing. + [Fact] + public async Task Without_a_registry_the_watcher_keeps_its_handles_private() + { + using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator()); + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + watcher.CurrentBindings.ShouldHaveSingleItem(); + (await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs new file mode 100644 index 00000000..8d48d6d5 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs @@ -0,0 +1,96 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-10: a dispatch-loop exit left the read loop draining into a channel +/// nobody read, with health Healthy and nothing telling the supervisor — a clean termination +/// evading the watchdog. The exit now reports a stream fault. The fault is injected through +/// the pump's handle factory, the one dispatch-side seam that runs outside the per-handler +/// isolation. +/// +public sealed class EventPumpDispatchFaultTests +{ + private const int WaitMs = 10_000; + + private sealed class ChannelSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + => Task.FromResult>([]); + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitAsync(int handle) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { DoubleValue = 1 }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + /// A dispatch-loop exception reports a stream fault carrying that exception. + [Fact] + public async Task Dispatch_loop_exit_reports_a_stream_fault() + { + var subscriber = new ChannelSubscriber(); + var registry = new SubscriptionRegistry(); + registry.Register(1, [new TagBinding("Tank01.Level", 7)]); + var fault = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var pump = new EventPump( + subscriber, registry, + handleFactory: _ => throw new InvalidDataException("synthetic dispatch failure"), + channelCapacity: 4, clientName: "DispatchFault", + onStreamFault: ex => fault.TrySetResult(ex)); + pump.Start(); + + await subscriber.EmitAsync(7); + + (await Task.WhenAny(fault.Task, Task.Delay(WaitMs))).ShouldBe(fault.Task, + "a dispatch-loop exit must be reported as a stream fault"); + (await fault.Task).ShouldBeOfType(); + } + + /// Positive control: the same event with a working handle factory dispatches and reports no fault. + [Fact] + public async Task Same_event_with_a_working_factory_dispatches_without_a_fault() + { + var subscriber = new ChannelSubscriber(); + var registry = new SubscriptionRegistry(); + registry.Register(1, [new TagBinding("Tank01.Level", 7)]); + var faulted = false; + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var pump = new EventPump( + subscriber, registry, channelCapacity: 4, clientName: "DispatchFault", + onStreamFault: _ => faulted = true); + pump.OnDataChange += (_, e) => delivered.TrySetResult(e); + pump.Start(); + + await subscriber.EmitAsync(7); + + (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task); + (await delivered.Task).FullReference.ShouldBe("Tank01.Level"); + faulted.ShouldBeFalse(); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs new file mode 100644 index 00000000..af842c93 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs @@ -0,0 +1,217 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-4 / G-7 / G-11: the per-platform ScanState probing was structurally +/// dead (its handles were never in the registry the pump resolves through, and a reconnect +/// never re-subscribed them), discovery fetched the full galaxy twice, and +/// LastSuccessfulRead never moved on data. These drive the real driver → pump → watcher +/// → aggregator path with a fake subscriber and a fake hierarchy. +/// +public sealed class GalaxyDriverProbeWiringTests +{ + private const int WaitMs = 10_000; + + private static GalaxyDriverOptions Opts() => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("ProbeWiring"), + new GalaxyRepositoryOptions(WatchDeployEvents: false), + new GalaxyReconnectOptions()); + + private sealed class CountingHierarchySource : IGalaxyHierarchySource + { + public int Calls; + + public Task> GetHierarchyAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref Calls); + var platform = new GalaxyObject { GobjectId = 1, TagName = "Plat01", ContainedName = "Plat01" }; + platform.TemplateChain.Add("$WinPlatform"); + var tank = new GalaxyObject { GobjectId = 2, TagName = "Tank01", ContainedName = "Tank01", ParentGobjectId = 1 }; + tank.TemplateChain.Add("$UserDefined"); + tank.Attributes.Add(new GalaxyAttribute { AttributeName = "Level", FullTagReference = "Tank01.Level", MxDataType = 5 }); + return Task.FromResult>([platform, tank]); + } + } + + private sealed class StreamingSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + private int _nextHandle; + + public List Subscribed { get; } = []; + public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var results = new List(); + lock (Subscribed) + { + foreach (var r in fullReferences) + { + var h = Interlocked.Increment(ref _nextHandle); + Subscribed.Add(r); + HandleByRef[r] = h; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + } + } + return Task.FromResult>(results); + } + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { BoolValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + + public ValueTask EmitDoubleAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { DoubleValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + private sealed class NullBuilder : IAddressSpaceBuilder + { + public IAddressSpaceBuilder Folder(string browseName, string displayName) => this; + public IVariableHandle Variable(string browseName, string displayName, DriverAttributeInfo attributeInfo) => new Handle(attributeInfo.FullName); + public void AddProperty(string browseName, DriverDataType dataType, object? value) { } + private sealed class Handle(string fullReference) : IVariableHandle + { + public string FullReference { get; } = fullReference; + public IAlarmConditionSink MarkAsAlarmCondition(AlarmConditionInfo alarmInfo) => new NoopSink(); + } + private sealed class NoopSink : IAlarmConditionSink + { + public void OnTransition(AlarmEventArgs args) { } + } + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } + + private static HostState? StateOf(GalaxyDriver driver, string host) => + driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; + + /// + /// A ScanState event on the probe handle updates GetHostStatuses; after a reconnect replay + /// the probe is re-subscribed and events on the NEW handle keep updating it, while the dead + /// pre-reconnect handle no longer does. Probe traffic never reaches public OnDataChange. + /// + [Fact] + public async Task ScanState_event_updates_GetHostStatuses_and_survives_a_reconnect() + { + var sub = new StreamingSubscriber(); + var source = new CountingHierarchySource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub); + var publicRefs = new List(); + driver.OnDataChange += (_, e) => { lock (publicRefs) publicRefs.Add(e.FullReference); }; + + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None); + // The pump starts on the first value subscription (the probe watcher does not start it). + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + + var probeHandle = sub.HandleByRef["Plat01.ScanState"]; + await sub.EmitBoolAsync(probeHandle, true); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, + "a ScanState=true event must surface Plat01 as Running"); + + // Reconnect replay: every pre-reconnect handle is dead; the probe must rebind. + await driver.InvokeReplayForTestAsync(CancellationToken.None); + var rebound = sub.HandleByRef["Plat01.ScanState"]; + rebound.ShouldNotBe(probeHandle, "replay must re-subscribe the probe and record the fresh handle"); + + await sub.EmitBoolAsync(rebound, false); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Stopped, + "a ScanState=false event on the post-reconnect handle must surface Plat01 as Stopped"); + + // The dead handle is no longer routed (positive control: the live one still is). + await sub.EmitBoolAsync(probeHandle, true); + await sub.EmitDoubleAsync(sub.HandleByRef["Tank01.Level"], 1.0); + await WaitUntilAsync(() => { lock (publicRefs) return publicRefs.Contains("Tank01.Level"); }, "value traffic must still flow"); + StateOf(driver, "Plat01").ShouldBe(HostState.Stopped, "an event on the pre-reconnect handle must be dropped"); + + lock (publicRefs) publicRefs.ShouldNotContain("Plat01.ScanState", "probe traffic is the watcher's, never the host's"); + } + + /// Discovery asks the gateway for the hierarchy once; the probe set is selected from that same sweep. + [Fact] + public async Task DiscoverAsync_fetches_the_hierarchy_once_and_still_syncs_the_probe_set() + { + var sub = new StreamingSubscriber(); + var source = new CountingHierarchySource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None); + + source.Calls.ShouldBe(1, "the probe selection must reuse the discoverer's sweep"); + sub.Subscribed.ShouldContain("Plat01.ScanState", "positive control: the platform probe was still synced"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]); + } + + /// Inbound stream data advances LastSuccessfulRead (throttled); the throttle holds the stamp still. + [Fact] + public async Task Stream_data_advances_LastSuccessfulRead_under_a_throttle() + { + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) + { + LivenessStampThrottle = TimeSpan.FromHours(1), + }; + await driver.InitializeAsync("{}", CancellationToken.None); + var initStamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + await Task.Delay(20); + + var delivered = 0; + driver.OnDataChange += (_, _) => Interlocked.Increment(ref delivered); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + var handle = sub.HandleByRef["Tank01.Level"]; + + await sub.EmitDoubleAsync(handle, 1.0); + await WaitUntilAsync(() => Volatile.Read(ref delivered) == 1, "first event must be delivered"); + var first = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + first.ShouldBeGreaterThan(initStamp, "data on the stream is liveness evidence"); + + await Task.Delay(20); + await sub.EmitDoubleAsync(handle, 2.0); + await WaitUntilAsync(() => Volatile.Read(ref delivered) == 2, "second event must be delivered"); + driver.GetHealth().LastSuccessfulRead.ShouldBe(first, "inside the throttle window the stamp holds still"); + } +} From 6dedf5158f0068fd99ed14ba94ea06ea312d0911 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:41:47 -0400 Subject: [PATCH 3/8] feat(galaxy): wire DiscoverPageSize and ProbeTimeoutSeconds; concurrent first-touch write lanes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review 2026-10-21 G-5, G-6, G-8 ("wire them all"). - G-5: GatewayGalaxyHierarchySource now pages the DiscoverHierarchy RPC itself (DiscoverHierarchyRawAsync) at Repository.DiscoverPageSize. The mxaccessgw client's typed DiscoverHierarchyAsync overloads hard-code a 5000-row page and DiscoverHierarchyOptions carries no page size, so the authored knob reached nothing; the raw request takes page_size on the wire, so it is honoured here (non-positive → 5000). The loop keeps the client's repeated-page-token guard. - G-6: GalaxyDriverProbe reads ProbeTimeoutSeconds from the config and runs under min(caller budget, configured cap) — the knob can shorten a probe, never extend it past the host's budget. The TCP preflight now shares that deadline instead of hanging on the caller's token alone. - G-8: GatewayGalaxyDataWriter runs a batch as one lane per distinct tag, concurrently and bounded at 8 (MaxConcurrentWriteLanes); same-tag writes keep their authored order inside one lane and results land at their request's index. Acks stay serial by design. - GalaxyDriverForm: help text now says what the two knobs actually do. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Browse/GatewayGalaxyHierarchySource.cs | 67 ++++++- .../GalaxyDriver.cs | 3 +- .../GalaxyDriverProbe.cs | 31 +++- .../Runtime/GatewayGalaxyDataWriter.cs | 78 +++++++- .../Drivers/Forms/GalaxyDriverForm.razor | 4 +- ...GatewayGalaxyHierarchySourcePagingTests.cs | 92 ++++++++++ .../GalaxyDriverProbeTimeoutTests.cs | 69 +++++++ ...GatewayGalaxyDataWriterConcurrencyTests.cs | 168 ++++++++++++++++++ 8 files changed, 498 insertions(+), 14 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs index bafabb59..68899bb8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs @@ -5,22 +5,83 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; /// /// Default wrapping the gateway's -/// . Pages internally via the client's overload. +/// . Pages the DiscoverHierarchy RPC itself, +/// at the configured Repository.DiscoverPageSize: the client's typed +/// DiscoverHierarchyAsync overloads hard-code a 5000-row page +/// (DiscoverHierarchyOptions carries no page size), so the knob the AdminUI +/// authors and the factory parses used to reach nothing. The raw RPC takes the page +/// size on the wire (DiscoverHierarchyRequest.page_size), so it is honoured here. /// public sealed class GatewayGalaxyHierarchySource : IGalaxyHierarchySource { + /// The page size used when the configured one is not positive — the client library's own default. + public const int DefaultPageSize = 5000; + private readonly GalaxyRepositoryClient _client; + private readonly int _pageSize; /// /// Initializes a new instance of the class. /// /// The gateway's Galaxy repository client for discovering the object hierarchy. - public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client) + /// + /// Objects per DiscoverHierarchy page (Repository.DiscoverPageSize). A value that is + /// not positive falls back to . + /// + public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client, int pageSize = DefaultPageSize) { _client = client ?? throw new ArgumentNullException(nameof(client)); + _pageSize = EffectivePageSize(pageSize); } + /// The page size this source sends on every DiscoverHierarchy request. + public int PageSize => _pageSize; + + /// Clamp a configured page size to something the gateway accepts: non-positive → . + /// The configured value. + /// The page size to send. + public static int EffectivePageSize(int configured) => configured > 0 ? configured : DefaultPageSize; + /// public Task> GetHierarchyAsync(CancellationToken cancellationToken) - => _client.DiscoverHierarchyAsync(cancellationToken); + => PageAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken); + + /// + /// Walk every DiscoverHierarchy page at and concatenate the + /// objects. Mirrors the client library's own loop (including its repeated-page-token guard, + /// which turns a misbehaving gateway into an error rather than an endless walk), with the + /// RPC behind a delegate so the loop is unit-testable without a channel. + /// + /// The raw DiscoverHierarchy RPC. + /// Objects per page; sent on every request. + /// Cancellation token for the walk. + /// Every object across all pages, in gateway order. + internal static async Task> PageAsync( + Func> discover, + int pageSize, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(discover); + var effective = EffectivePageSize(pageSize); + + var objects = new List(); + var seenPageTokens = new HashSet(StringComparer.Ordinal); + var pageToken = string.Empty; + do + { + var request = new DiscoverHierarchyRequest { PageSize = effective, PageToken = pageToken }; + var reply = await discover(request, cancellationToken).ConfigureAwait(false); + + objects.AddRange(reply.Objects); + pageToken = reply.NextPageToken ?? string.Empty; + if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken)) + { + throw new InvalidOperationException( + $"Galaxy DiscoverHierarchy returned a repeated page token '{pageToken}' — refusing to loop."); + } + } + while (!string.IsNullOrWhiteSpace(pageToken)); + + return objects; + } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index f2768291..8cb4bcb5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -1821,7 +1821,8 @@ public sealed class GalaxyDriver _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); return new TracedGalaxyHierarchySource( - new GatewayGalaxyHierarchySource(_ownedRepositoryClient), _options.MxAccess.ClientName); + new GatewayGalaxyHierarchySource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize), + _options.MxAccess.ClientName); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs index a4010d07..81807500 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs @@ -54,12 +54,18 @@ public sealed class GalaxyDriverProbe : IDriverProbe if (string.IsNullOrWhiteSpace(host) || port <= 0) return new(false, "Config has no host/port to probe.", null); + // The config's own ProbeTimeoutSeconds caps the probe deadline. It can only tighten the + // caller's budget, never extend it — the caller's token fires at `timeout` regardless. + timeout = EffectiveTimeout(timeout, opts.ProbeTimeoutSeconds); + using var probeCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + probeCts.CancelAfter(timeout); + // --- TCP preflight: fast-fail for closed ports / unreachable hosts --- var sw = Stopwatch.StartNew(); try { using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - await socket.ConnectAsync(host, port, ct); + await socket.ConnectAsync(host, port, probeCts.Token); } catch (SocketException ex) { @@ -86,10 +92,7 @@ public sealed class GalaxyDriverProbe : IDriverProbe { client = GalaxyRepositoryClient.Create(BuildProbeClientOptions(opts.Gateway, timeout)); - using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct); - deadlineCts.CancelAfter(timeout); - - await client.TestConnectionAsync(deadlineCts.Token).ConfigureAwait(false); + await client.TestConnectionAsync(probeCts.Token).ConfigureAwait(false); sw.Stop(); // No RpcException ⇒ the gateway answered the gRPC call successfully. @@ -157,6 +160,24 @@ public sealed class GalaxyDriverProbe : IDriverProbe _ => (false, $"Reachable at {host}:{port} but gateway gRPC handshake failed: {code}"), }; + /// + /// The deadline the probe actually runs under: the caller's budget, tightened by the + /// config's when that is positive and + /// shorter. The knob was authored by the AdminUI and parsed by the factory but consumed by + /// nothing — the probe always ran on the caller's budget alone. A non-positive caller budget + /// is treated as the config value (or 1 s when that is unset too), as before. + /// + /// The budget the host handed the probe. + /// The config's ProbeTimeoutSeconds; ignored when not positive. + /// The effective probe deadline. + internal static TimeSpan EffectiveTimeout(TimeSpan callerTimeout, int configuredSeconds) + { + var caller = callerTimeout > TimeSpan.Zero ? callerTimeout : TimeSpan.FromSeconds(1); + if (configuredSeconds <= 0) return caller; + var configured = TimeSpan.FromSeconds(configuredSeconds); + return configured < caller ? configured : caller; + } + /// /// Builds the gRPC client options for the probe ping from the gateway config WITHOUT /// resolving the API-key secret ref (the host owns secret resolution; the probe sends the diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs index a472f269..957459df 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs @@ -120,15 +120,87 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter // Bind the per-batch MX write operations behind the internal IMxWriteOps seam (see below) so the // whole write pipeline is unit-testable without the sealed SDK session (archreview 06/S-1). var ops = new SessionMxWriteOps(session, serverHandle); + return await WriteBatchAsync(ops, writes, securityResolver, cancellationToken).ConfigureAwait(false); + } + /// + /// Upper bound on write lanes in flight at once. A first-touch write is three round-trips + /// (AddItem → AdviseSupervisory → Write) and the batch used to run them strictly serially; + /// distinct tags are independent, so their lanes now overlap, bounded so a large batch cannot + /// flood the gateway's STA pump. Acks stay serial by design (see GalaxyDriver.AcknowledgeAsync). + /// + internal const int MaxConcurrentWriteLanes = 8; + + /// Test seam: drive the real batch pipeline against a fake . + /// The (fake) MX write operations. + /// The batch. + /// Per-tag classification resolver. + /// The cancellation token. + /// One result per request, in input order. + internal Task> WriteBatchForTestAsync( + IMxWriteOps ops, IReadOnlyList writes, + Func securityResolver, CancellationToken ct) + => WriteBatchAsync(ops, writes, securityResolver, ct); + + /// + /// Run a batch as one lane per distinct tag (OrdinalIgnoreCase): writes to the SAME tag + /// stay in input order inside their lane, lanes run concurrently up to + /// , and results land at their request's index. + /// A single-tag batch takes the direct path with no gate. + /// + private async Task> WriteBatchAsync( + IMxWriteOps ops, IReadOnlyList writes, + Func securityResolver, CancellationToken ct) + { var results = new WriteResult[writes.Count]; + if (writes.Count == 0) return results; + + // Group request indices by tag, first-seen order preserved so the lane start order is + // deterministic and a same-tag sequence keeps its authored order. + var lanes = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var laneOrder = new List>(); for (var i = 0; i < writes.Count; i++) { - results[i] = await WriteOneAsync(ops, writes[i], - securityResolver(writes[i].FullReference), cancellationToken) - .ConfigureAwait(false); + if (!lanes.TryGetValue(writes[i].FullReference, out var lane)) + { + lane = []; + lanes[writes[i].FullReference] = lane; + laneOrder.Add(lane); + } + lane.Add(i); } + + if (laneOrder.Count == 1) + { + await RunLaneAsync(laneOrder[0]).ConfigureAwait(false); + return results; + } + + using var gate = new SemaphoreSlim(MaxConcurrentWriteLanes, MaxConcurrentWriteLanes); + var tasks = new Task[laneOrder.Count]; + for (var l = 0; l < laneOrder.Count; l++) + { + var lane = laneOrder[l]; + tasks[l] = RunGatedAsync(lane); + } + await Task.WhenAll(tasks).ConfigureAwait(false); return results; + + async Task RunGatedAsync(List lane) + { + await gate.WaitAsync(ct).ConfigureAwait(false); + try { await RunLaneAsync(lane).ConfigureAwait(false); } + finally { gate.Release(); } + } + + async Task RunLaneAsync(List lane) + { + foreach (var i in lane) + { + results[i] = await WriteOneAsync(ops, writes[i], + securityResolver(writes[i].FullReference), ct).ConfigureAwait(false); + } + } } /// Test seam: drive the real single-write pipeline against a fake diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor index 17468d7d..705f70c7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor @@ -92,7 +92,7 @@
-
Default 5000 objects per page.
+
Objects per DiscoverHierarchy page the driver asks the gateway for. Default 5000.
@@ -138,7 +138,7 @@
-
Max 60. Used by Test Connect. Default 30.
+
Caps the Test Connect probe deadline (the probe's own budget still applies, so this only shortens it). Max 60. Default 30.
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs new file mode 100644 index 00000000..d0767592 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs @@ -0,0 +1,92 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse; + +/// +/// Review 2026-10-21 G-5: Repository.DiscoverPageSize was authored and parsed but reached +/// nothing — the client's typed DiscoverHierarchyAsync hard-codes 5000. The source now pages +/// the raw RPC itself at the configured size. +/// +public sealed class GatewayGalaxyHierarchySourcePagingTests +{ + private static GalaxyObject Obj(int id) => new() { GobjectId = id, TagName = $"Obj{id}" }; + + /// Every request carries the configured page size and the page tokens chain until the last page. + [Fact] + public async Task Pages_at_the_configured_size_and_chains_tokens() + { + var requests = new List(); + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + requests.Add(req); + var reply = new DiscoverHierarchyReply(); + switch (req.PageToken) + { + case "": + reply.Objects.Add([Obj(1), Obj(2)]); + reply.NextPageToken = "p2"; + break; + case "p2": + reply.Objects.Add([Obj(3), Obj(4)]); + reply.NextPageToken = "p3"; + break; + default: + reply.Objects.Add(Obj(5)); + reply.NextPageToken = ""; + break; + } + return Task.FromResult(reply); + } + + var objects = await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 2, CancellationToken.None); + + objects.Select(o => o.GobjectId).ShouldBe([1, 2, 3, 4, 5]); + requests.Count.ShouldBe(3); + requests.ShouldAllBe(r => r.PageSize == 2); + requests.Select(r => r.PageToken).ShouldBe(["", "p2", "p3"]); + } + + /// A different configured size is what goes on the wire (positive control for the knob). + [Fact] + public async Task A_different_page_size_is_sent_on_the_wire() + { + int? sent = null; + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + sent = req.PageSize; + return Task.FromResult(new DiscoverHierarchyReply()); + } + + await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 250, CancellationToken.None); + sent.ShouldBe(250); + } + + /// A non-positive configured size falls back to the client library's 5000 default. + [Theory] + [InlineData(0)] + [InlineData(-7)] + public void Non_positive_page_size_falls_back_to_the_default(int configured) + => GatewayGalaxyHierarchySource.EffectivePageSize(configured).ShouldBe(GatewayGalaxyHierarchySource.DefaultPageSize); + + /// A repeated page token is refused rather than walked forever. + [Fact] + public async Task A_repeated_page_token_is_refused() + { + var calls = 0; + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + calls++; + var reply = new DiscoverHierarchyReply { NextPageToken = "same" }; + reply.Objects.Add(Obj(calls)); + return Task.FromResult(reply); + } + + var ex = await Should.ThrowAsync( + () => GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 10, CancellationToken.None)); + ex.Message.ShouldContain("repeated page token"); + calls.ShouldBe(2); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs new file mode 100644 index 00000000..1f5fbb78 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs @@ -0,0 +1,69 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests; + +/// +/// Review 2026-10-21 G-6: ProbeTimeoutSeconds was authored and compared for equivalence but +/// never consumed — the probe ran on the caller's budget alone. It now caps the probe deadline +/// (it can shorten the caller's budget, never extend it). +/// +[Trait("Category", "Unit")] +public sealed class GalaxyDriverProbeTimeoutTests +{ + /// The effective deadline is the shorter of the caller's budget and the configured cap. + [Theory] + [InlineData(10, 30, 10)] // caller shorter → caller wins + [InlineData(10, 2, 2)] // config shorter → config caps + [InlineData(10, 0, 10)] // unset config → caller + [InlineData(10, -1, 10)] // nonsense config → caller + [InlineData(0, 5, 1)] // no caller budget → 1 s floor beats a longer config + [InlineData(0, 0, 1)] // nothing → 1 s floor + public void EffectiveTimeout_is_the_shorter_of_caller_and_config(int callerSeconds, int configSeconds, int expectedSeconds) + => GalaxyDriverProbe.EffectiveTimeout(TimeSpan.FromSeconds(callerSeconds), configSeconds) + .ShouldBe(TimeSpan.FromSeconds(expectedSeconds)); + + /// + /// Against a port that accepts TCP but never speaks gRPC, a 1 s configured cap returns long + /// before the caller's 20 s budget. The gRPC handshake itself does the waiting here, so this + /// exercises the real deadline path, not the pure helper. + /// + [Fact] + public async Task Configured_cap_shortens_the_probe_against_a_silent_listener() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var config = $$""" + { + "Gateway": { "Endpoint": "http://127.0.0.1:{{port}}", "ApiKeySecretRef": "dev:k", "UseTls": false }, + "MxAccess": { "ClientName": "probe-timeout" }, + "Repository": {}, + "Reconnect": {}, + "ProbeTimeoutSeconds": 1 + } + """; + + var sw = Stopwatch.StartNew(); + var result = await new GalaxyDriverProbe().ProbeAsync(config, TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken); + sw.Stop(); + + result.Ok.ShouldBeFalse(); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), $"the 1 s configured cap must bound the probe (message: {result.Message})"); + result.Message.ShouldNotBeNull(); + (result.Message!.Contains("timed out", StringComparison.OrdinalIgnoreCase) + || result.Message.Contains("DeadlineExceeded", StringComparison.OrdinalIgnoreCase) + || result.Message.Contains("handshake failed", StringComparison.OrdinalIgnoreCase)) + .ShouldBeTrue($"unexpected message: '{result.Message}'"); + } + finally + { + listener.Stop(); + } + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs new file mode 100644 index 00000000..dff0f665 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs @@ -0,0 +1,168 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-8: a batch of writes ran strictly serially, three round-trips per +/// first-touch tag. Distinct tags now run as concurrent lanes bounded at +/// ; same-tag writes keep their +/// order inside one lane; results stay at their request's index. +/// +public sealed class GatewayGalaxyDataWriterConcurrencyTests +{ + private const int WaitMs = 10_000; + + private static GatewayGalaxyDataWriter NewWriter() + => new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0); + + /// + /// Fake ops whose AddItem parks every caller until callers are in + /// flight at once (or a safety delay elapses), recording the peak concurrency. + /// + private sealed class GatingOps : IMxWriteOps + { + private int _inFlight; + private int _peak; + private int _nextHandle; + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ConcurrentDictionary _handles = new(StringComparer.OrdinalIgnoreCase); + + public int ReleaseAt { get; init; } = int.MaxValue; + public int Peak => Volatile.Read(ref _peak); + public ConcurrentQueue<(int Handle, MxValue Value)> RawWrites { get; } = new(); + public ConcurrentQueue AddItems { get; } = new(); + public Func? WriteResponder { get; init; } + + /// Deterministic handle per tag when set; otherwise handles are assigned in AddItem completion order. + public Func? HandleFor { get; init; } + + public string SessionId => "sess"; + public int ServerHandle => 1; + + public async Task AddItemAsync(string fullRef, CancellationToken ct) + { + AddItems.Enqueue(fullRef); + var now = Interlocked.Increment(ref _inFlight); + int seen; + do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen); + if (now >= ReleaseAt) _release.TrySetResult(); + await Task.WhenAny(_release.Task, Task.Delay(300, ct)); + Interlocked.Decrement(ref _inFlight); + return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle)); + } + + public Task InvokeAsync(MxCommandRequest request, CancellationToken ct) + => Task.FromResult(new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } }); + + public Task WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct) + { + RawWrites.Enqueue((itemHandle, value)); + return Task.FromResult(WriteResponder?.Invoke(itemHandle) ?? GoodStatusRow()); + } + + // A reply carrying a Good MX status ROW (not an empty statuses array): the writer's static + // meters are process-wide, and an empty-statuses Good would increment galaxy.writes.unconfirmed + // under the meter tests running in parallel in another class. + private static MxCommandReply GoodStatusRow() + { + var reply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } }; + reply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok }); + return reply; + } + } + + private static SecurityClassification Free(string _) => SecurityClassification.FreeAccess; + + /// Sixteen distinct tags: lanes overlap (not serial) and never exceed the cap of eight. + [Fact] + public async Task Distinct_tags_write_concurrently_bounded_at_the_lane_cap() + { + var ops = new GatingOps { ReleaseAt = GatewayGalaxyDataWriter.MaxConcurrentWriteLanes }; + var writes = Enumerable.Range(0, 16).Select(i => new WriteRequest($"Tag{i}.PV", (double)i)).ToArray(); + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + ops.Peak.ShouldBe(GatewayGalaxyDataWriter.MaxConcurrentWriteLanes, + "first-touch lanes must overlap (serial = 1) and stay bounded (unbounded > 8)"); + results.Count.ShouldBe(16); + results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good); + ops.AddItems.Count.ShouldBe(16, "one AddItem per distinct tag"); + ops.RawWrites.Count.ShouldBe(16); + } + + /// Writes to the same tag stay in input order inside their lane and share one AddItem. + [Fact] + public async Task Same_tag_writes_stay_ordered_and_share_one_handle() + { + var ops = new GatingOps { ReleaseAt = 2 }; + var writes = new[] + { + new WriteRequest("Tank01.SP", 1.0), + new WriteRequest("Tank02.SP", 10.0), + new WriteRequest("Tank01.SP", 2.0), + new WriteRequest("Tank01.SP", 3.0), + }; + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + results.Count.ShouldBe(4); + results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good); + ops.AddItems.Count(a => a == "Tank01.SP").ShouldBe(1, "a lane AddItems its tag once"); + var tank01 = ops.RawWrites.Where(w => w.Handle == ops.RawWrites.First(x => x.Value.DoubleValue == 1.0).Handle) + .Select(w => w.Value.DoubleValue).ToArray(); + tank01.ShouldBe([1.0, 2.0, 3.0], "same-tag writes must keep their authored order"); + } + + /// A per-tag failure lands at that request's index while the other lanes stay Good. + [Fact] + public async Task Results_land_at_their_request_index() + { + var ops = new GatingOps + { + ReleaseAt = 3, + HandleFor = tag => tag[0] - 'A' + 1, // A.PV → 1, B.PV → 2, C.PV → 3 + WriteResponder = handle => new MxCommandReply + { + ProtocolStatus = new ProtocolStatus + { + Code = handle == 2 ? ProtocolStatusCode.MxaccessFailure : ProtocolStatusCode.Ok, + }, + }, + }; + var writes = new[] + { + new WriteRequest("A.PV", 1.0), + new WriteRequest("B.PV", 2.0), // refused by the gateway + new WriteRequest("C.PV", 3.0), + }; + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + results.Select(r => r.StatusCode).ShouldBe( + [StatusCodeMap.Good, StatusCodeMap.BadCommunicationError, StatusCodeMap.Good], + "each result must sit at its request's index regardless of lane completion order"); + } + + /// A single-tag batch takes the direct path and still produces one result per request. + [Fact] + public async Task Single_tag_batch_runs_serially_in_order() + { + var ops = new GatingOps { ReleaseAt = 1 }; + var writes = new[] { new WriteRequest("X.PV", 1.0), new WriteRequest("X.PV", 2.0) }; + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + results.Count.ShouldBe(2); + ops.Peak.ShouldBe(1); + ops.RawWrites.Select(w => w.Value.DoubleValue).ShouldBe([1.0, 2.0]); + } +} From 05a0e51fc1a237c11fea22695289adf738a58d50 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:29:11 -0400 Subject: [PATCH 4/8] fix(galaxy): GalaxyMxSession.CloseAsync(ct) bounds the gateway CloseSession The SDK's DisposeAsync closes with no token, i.e. under the 30 s DefaultCallTimeout, so a blackholed gateway parked the driver's teardown for the whole of it. CloseAsync(ct) runs the close under the caller's token, drops the channel on a cancelled/failed close (the gateway reaps the session on its idle lease) and leaves the session re-openable. A close followed by the final dispose closes the gateway session once. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Runtime/GalaxyMxSession.cs | 60 ++++++++++--- .../Runtime/GalaxyMxSessionCloseTests.cs | 90 +++++++++++++++++++ 2 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs index 0daae292..75550740 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs @@ -60,6 +60,12 @@ public sealed class GalaxyMxSession : IAsyncDisposable /// internal Func? OpenAndRegisterOverrideForTests { get; set; } + /// + /// Test seam — replaces the gateway CloseSession round-trip in + /// so the bounded-close behaviour can be exercised against a never-answering fake. + /// + internal Func? CloseOverrideForTests { get; set; } + /// /// Connect the underlying gateway client + open an MXAccess session + register the /// configured client name. Idempotent — second calls are no-ops while a connect has @@ -96,7 +102,7 @@ public sealed class GalaxyMxSession : IAsyncDisposable { // A partial open (e.g. OpenSession succeeded but Register threw) must not leave a half-open // handle that blocks the next reconnect. Tear the partial state down so a retry rebuilds cleanly. - await TeardownSessionAsync().ConfigureAwait(false); + await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false); throw; } } @@ -112,10 +118,23 @@ public sealed class GalaxyMxSession : IAsyncDisposable public async Task RecreateAsync(MxGatewayClientOptions clientOptions, CancellationToken cancellationToken) { ObjectDisposedException.ThrowIf(_disposed, this); - await TeardownSessionAsync().ConfigureAwait(false); + await TeardownSessionAsync(cancellationToken).ConfigureAwait(false); await ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false); } + /// + /// Close the gateway session and release the owned client under + /// . The SDK's DisposeAsync closes with NO token, + /// i.e. under the 30 s DefaultCallTimeout — against a blackholed gateway that parks + /// the driver's ShutdownAsync (and the host's PostStop) for the full timeout. + /// A close that is cancelled or fails is NOT retried through the SDK dispose (which would + /// park again); the channel is dropped and the gateway reaps the session on its idle lease. + /// The session can be re-opened afterwards (this does not set _disposed). + /// + /// Bounds the gateway CloseSession round-trip. + /// A task that completes once the session and client are released. + public Task CloseAsync(CancellationToken cancellationToken) => TeardownSessionAsync(cancellationToken); + /// /// Test seam — attach a session opened externally (e.g. against an in-process gw /// fake). Skips the gateway-client construction so tests can drive the session @@ -143,24 +162,45 @@ public sealed class GalaxyMxSession : IAsyncDisposable { if (_disposed) return; _disposed = true; - await TeardownSessionAsync().ConfigureAwait(false); + await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false); } /// - /// Resets the connect flag and best-effort disposes + nulls the owned session and client. - /// Shared by (final teardown) and / - /// the partial-open recovery path (rebuild teardown) — it does NOT set _disposed, so - /// a torn-down session can be re-opened. + /// Resets the connect flag and best-effort closes + nulls the owned session and client. + /// Shared by (final teardown), (bounded + /// teardown) and / the partial-open recovery path (rebuild + /// teardown) — it does NOT set _disposed, so a torn-down session can be re-opened. /// - private async Task TeardownSessionAsync() + private async Task TeardownSessionAsync(CancellationToken cancellationToken) { + var wasConnected = _connected; _connected = false; + // The override models the gateway round-trip, so it runs only when there was a live + // session to close — a second teardown (CloseAsync then DisposeAsync) must not close twice. + var closeOverride = CloseOverrideForTests; + if (closeOverride is not null && wasConnected) + { + try { await closeOverride(cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession close failed or timed out (best-effort)"); } + } + if (_session is not null) { - try { await _session.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession session dispose failed (best-effort)"); } + var session = _session; _session = null; + try + { + // CloseAsync(ct) is the bounded gateway round-trip; DisposeAsync only re-runs an + // un-replied close (with no token), so it is reached only after a successful one. + await session.CloseAsync(cancellationToken).ConfigureAwait(false); + await session.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyMxSession close failed or timed out (best-effort) — dropping the channel; the gateway reaps the session on its idle lease."); + } } if (_ownedClient is not null) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs new file mode 100644 index 00000000..aa6219c3 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs @@ -0,0 +1,90 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// bounds the gateway CloseSession round-trip by +/// the caller's token (the SDK dispose closes under its 30 s call timeout) and leaves the session +/// re-openable. +/// +public sealed class GalaxyMxSessionCloseTests +{ + /// A close against a never-answering gateway returns when the token fires, without throwing. + [Fact] + public async Task CloseAsync_returns_at_the_token_deadline_against_a_never_answering_gateway() + { + var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = ct => Task.Delay(Timeout.Infinite, ct), + }; + await session.ConnectAsync(null!, CancellationToken.None); + session.IsConnected.ShouldBeTrue(); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + var sw = System.Diagnostics.Stopwatch.StartNew(); + await session.CloseAsync(cts.Token).WaitAsync(TimeSpan.FromSeconds(10)); + sw.Stop(); + + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5)); + session.IsConnected.ShouldBeFalse(); + + // Still usable: a close is not a dispose. + await session.ConnectAsync(null!, CancellationToken.None); + session.IsConnected.ShouldBeTrue(); + } + + /// Positive control: a close that answers completes and reports the token it was handed. + [Fact] + public async Task CloseAsync_completes_when_the_gateway_answers() + { + CancellationToken seen = default; + var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = ct => { seen = ct; return Task.CompletedTask; }, + }; + await session.ConnectAsync(null!, CancellationToken.None); + + using var cts = new CancellationTokenSource(); + await session.CloseAsync(cts.Token); + + seen.ShouldBe(cts.Token, "the caller's token must reach the gateway round-trip"); + session.IsConnected.ShouldBeFalse(); + } + + /// + /// The driver's teardown is CloseAsync (bounded) then DisposeAsync (final). The dispose must + /// not issue a second gateway close; a fresh connect after a close must close again (positive + /// control: the guard keys on "was connected", not on "ever closed"). + /// + [Fact] + public async Task CloseAsync_then_DisposeAsync_closes_the_gateway_session_once() + { + var closes = 0; + var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; }, + }; + await session.ConnectAsync(null!, CancellationToken.None); + + await session.CloseAsync(CancellationToken.None); + await session.DisposeAsync(); + closes.ShouldBe(1, "dispose after a close must not close the gateway session again"); + + var reopened = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; }, + }; + await reopened.ConnectAsync(null!, CancellationToken.None); + await reopened.CloseAsync(CancellationToken.None); + await reopened.ConnectAsync(null!, CancellationToken.None); + await reopened.DisposeAsync(); + closes.ShouldBe(3, "each live session closes exactly once"); + } +} From 3456756cd1cf95fa4fd16dc83a67e59d982d8b1f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:29:11 -0400 Subject: [PATCH 5/8] fix(galaxy): seed the probe set at init, refuse pumps mid-teardown, bound the close Review fix round 1 for lane F: - The ScanState probe watcher is seeded at InitializeAsync from a platform-only DiscoverHierarchy query (IGalaxyPlatformSource / GatewayGalaxyPlatformSource over the owned repository client) and re-seeded on every deploy-watcher rediscovery. The runtime never calls DiscoverAsync on a live driver, so per-platform host status was unreachable in production. A failed fetch logs a Warning and never fails init; re-seeds coalesce so a deploy during an in-flight pass still gets a pass that starts after it. - EnsureEventPumpStarted refuses while the runtime is stopping, BEFORE reusing the live pump (that pump belongs to the dying subscriber), and the production rebuild disposes any stale pump before wiring the new subscriber. - TeardownRuntimeAsync closes the MX session under the caller's token linked with a 5 s SessionCloseDeadline, so a blackholed gateway cannot hold DriverInstanceActor.PostStop for 30 s. - A runtime generation counter, bumped by every teardown, is re-checked before each Rebind in ResubscribeAllAsync, so a replay-gap re-snapshot that straddles a teardown cannot resurrect handles into the cleared registry. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Browse/GatewayGalaxyPlatformSource.cs | 88 ++++ .../Browse/IGalaxyPlatformSource.cs | 18 + .../GalaxyDriver.cs | 185 +++++++- .../GatewayGalaxyPlatformSourceTests.cs | 60 +++ .../Runtime/GalaxyDriverFixRound1Tests.cs | 396 ++++++++++++++++++ .../Runtime/GalaxyDriverShutdownTests.cs | 18 +- 6 files changed, 754 insertions(+), 11 deletions(-) create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs new file mode 100644 index 00000000..dfed1659 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs @@ -0,0 +1,88 @@ +using ZB.MOM.WW.MxGateway.Client; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +/// +/// Default : a platform-only DiscoverHierarchy sweep +/// over the gateway's — the request filters on +/// template_chain_contains ($WinPlatform, $AppEngine) and asks for no +/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the +/// address-space discovery pulls. Paged at the configured Repository.DiscoverPageSize. +/// +public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource +{ + /// Template-chain tokens that identify a platform / engine object. + public static readonly IReadOnlyList PlatformTemplates = ["$WinPlatform", "$AppEngine"]; + + private readonly GalaxyRepositoryClient _client; + private readonly int _pageSize; + + /// Initializes a new instance of the class. + /// The gateway's Galaxy repository client. + /// Objects per page (Repository.DiscoverPageSize); non-positive → the client default. + public GatewayGalaxyPlatformSource(GalaxyRepositoryClient client, int pageSize = GatewayGalaxyHierarchySource.DefaultPageSize) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _pageSize = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize); + } + + /// + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken); + + /// + /// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every + /// request carries the template filter and include_attributes = false; the reply is + /// re-filtered on the returned template chain (a gateway that ignores the filter still + /// yields only platforms) and reduced to non-empty tag names. + /// + /// The raw DiscoverHierarchy RPC. + /// Objects per page. + /// Cancellation token for the walk. + /// Distinct platform tag names in gateway order. + internal static async Task> QueryAsync( + Func> discover, + int pageSize, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(discover); + var effective = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize); + + var names = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var seenPageTokens = new HashSet(StringComparer.Ordinal); + var pageToken = string.Empty; + do + { + var request = new DiscoverHierarchyRequest + { + PageSize = effective, + PageToken = pageToken, + IncludeAttributes = false, + }; + request.TemplateChainContains.Add(PlatformTemplates); + + var reply = await discover(request, cancellationToken).ConfigureAwait(false); + foreach (var obj in reply.Objects) + { + if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue; + if (seen.Add(obj.TagName)) names.Add(obj.TagName); + } + + pageToken = reply.NextPageToken ?? string.Empty; + if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken)) + { + throw new InvalidOperationException( + $"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop."); + } + } + while (!string.IsNullOrWhiteSpace(pageToken)); + + return names; + } + + private static bool IsPlatform(GalaxyObject obj) => + obj.TemplateChain.Count == 0 // a gateway that strips the chain already filtered server-side + || obj.TemplateChain.Any(t => PlatformTemplates.Contains(t, StringComparer.OrdinalIgnoreCase)); +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs new file mode 100644 index 00000000..6589c9aa --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs @@ -0,0 +1,18 @@ +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +/// +/// Driver-side seam for the set of Galaxy platform / engine objects whose ScanState +/// the PerPlatformProbeWatcher advises. Production wraps the gateway's +/// DiscoverHierarchy RPC filtered to $WinPlatform / $AppEngine with no +/// attributes (); tests substitute a canned list. +/// The driver seeds the watcher from this at InitializeAsync and re-seeds on every +/// deploy-watcher rediscovery — the runtime never calls DiscoverAsync on a live driver, +/// so without this seed the probe watcher never had a platform set in production. +/// +public interface IGalaxyPlatformSource +{ + /// The tag_name of every deployed $WinPlatform / $AppEngine object. + /// Cancellation token for the query. + /// Platform tag names; empty when the galaxy has none. + Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken); +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 8cb4bcb5..8fc00fa8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -150,6 +150,31 @@ public sealed class GalaxyDriver // GatewayGalaxyDeployWatchSource over the owned GalaxyRepositoryClient. private readonly IGalaxyDeployWatchSource? _deployWatchSource; + // Platform set for the ScanState probe watcher. Injected seam (tests) or, on the production + // path, a platform-only DiscoverHierarchy query over the owned repository client. Seeded at + // InitializeAsync and re-seeded on every deploy-watcher rediscovery: the runtime never calls + // DiscoverAsync on a live driver (only the /raw picker's throwaway does), so SyncPlatformsAsync + // from DiscoverAsync alone left the production watcher with no platforms, ever. + private readonly IGalaxyPlatformSource? _platformSource; + private IGalaxyPlatformSource? _ownedPlatformSource; + // Rediscovery re-seed requests not yet served. The first request starts the worker; later ones + // only bump the count, and the worker loops until it has served every request it saw — so a + // deploy that lands while a pass is mid-fetch still gets a pass that starts after it. + private int _reseedRequests; + + // Bumped by every runtime teardown. Background work started under one runtime (the replay-gap + // re-snapshot, a rediscovery re-seed) re-checks it before touching the registry / watcher, so a + // late Rebind cannot resurrect handles into a registry the teardown already cleared. + private int _runtimeGeneration; + + /// + /// Upper bound on the gateway CloseSession round-trip during a teardown. The SDK's + /// default close runs under the 30 s call timeout, which against a blackholed gateway parks + /// ShutdownAsync — and the host's PostStop — for the whole of it. Internal so + /// tests can shrink it. + /// + internal TimeSpan SessionCloseDeadline { get; set; } = TimeSpan.FromSeconds(5); + // True while the seams (_subscriber / _dataWriter / _hierarchySource / _alarmFeed / // _alarmAcknowledger) were built by BuildProductionRuntimeAsync and are therefore ours to dispose // and null on ShutdownAsync — the host's comms-loss recovery is ShutdownAsync → InitializeAsync on @@ -258,6 +283,11 @@ public sealed class GalaxyDriver /// Optional deploy-event stream seam. When supplied, starts the /// over it (production builds one over the owned repository client). /// + /// + /// Optional platform-set seam for the ScanState probe watcher. When supplied, + /// seeds the watcher from it and every rediscovery re-seeds + /// (production builds a over the owned repository client). + /// internal GalaxyDriver( string driverInstanceId, GalaxyDriverOptions options, @@ -269,9 +299,11 @@ public sealed class GalaxyDriver IGalaxyAlarmFeed? alarmFeed = null, ILogger? logger = null, ISecretResolver? secretResolver = null, - IGalaxyDeployWatchSource? deployWatchSource = null) + IGalaxyDeployWatchSource? deployWatchSource = null, + IGalaxyPlatformSource? platformSource = null) { _deployWatchSource = deployWatchSource; + _platformSource = platformSource; _driverInstanceId = !string.IsNullOrWhiteSpace(driverInstanceId) ? driverInstanceId : throw new ArgumentException("Driver instance id required.", nameof(driverInstanceId)); @@ -443,6 +475,7 @@ public sealed class GalaxyDriver } await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false); + await SeedProbePlatformsAsync("initialize", cancellationToken).ConfigureAwait(false); _logger.LogInformation( "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName); @@ -454,6 +487,92 @@ public sealed class GalaxyDriver } } + /// + /// Fetch the galaxy's platform / engine set and hand it to the probe watcher. Best-effort: + /// a failed fetch logs a Warning and leaves the watcher as it was — it must never fail an + /// InitializeAsync (the value path does not depend on it), and the next rediscovery + /// re-seeds. No-op when there is no watcher or no platform source (seam mode without one). + /// + private async Task SeedProbePlatformsAsync(string reason, CancellationToken cancellationToken) + { + var watcher = _probeWatcher; + if (watcher is null) return; + + IGalaxyPlatformSource? source; + try + { + source = await ResolvePlatformSourceAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} could not build the platform query ({Reason}) — per-platform host status stays unseeded until the next rediscovery.", + _driverInstanceId, reason); + return; + } + if (source is null) return; + + try + { + var platforms = await source.GetPlatformTagNamesAsync(cancellationToken).ConfigureAwait(false); + await watcher.SyncPlatformsAsync(platforms, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "GalaxyDriver {InstanceId} probe watcher seeded with {Count} platform(s) ({Reason}).", + _driverInstanceId, platforms.Count, reason); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} platform seed failed ({Reason}) — per-platform host status stays as it was until the next rediscovery.", + _driverInstanceId, reason); + } + } + + private async Task ResolvePlatformSourceAsync(CancellationToken cancellationToken) + { + if (_platformSource is not null) return _platformSource; + if (!_ownsRuntime) return null; // seam mode without an injected source: nothing to query + if (_ownedPlatformSource is not null) return _ownedPlatformSource; + + _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( + await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + return _ownedPlatformSource = + new GatewayGalaxyPlatformSource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize); + } + + /// + /// Deploy-watcher rediscovery: re-seed the platform set off the watcher's thread, one pass at + /// a time, and only if the runtime it was raised under is still the live one. + /// + private void ReseedProbePlatformsOnRediscovery() + { + if (_runtimeStopping || _probeWatcher is null) return; + if (Interlocked.Increment(ref _reseedRequests) != 1) return; // the running worker will serve it + var generation = Volatile.Read(ref _runtimeGeneration); + + _ = Task.Run(async () => + { + int served; + do + { + served = Volatile.Read(ref _reseedRequests); + try + { + if (Volatile.Read(ref _runtimeGeneration) == generation && !_runtimeStopping) + { + await SeedProbePlatformsAsync("rediscovery", CancellationToken.None).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "GalaxyDriver {InstanceId} rediscovery re-seed failed.", _driverInstanceId); + } + } + while (Interlocked.Add(ref _reseedRequests, -served) != 0); + }); + } + // The watcher publishes its probe handles into the driver's registry (reserved id) so the // EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every // probe event was dropped before it got anywhere near the watcher. @@ -488,6 +607,18 @@ public sealed class GalaxyDriver private async Task BuildProductionRuntimeCoreAsync(CancellationToken cancellationToken) { + // A pump that slipped in between a teardown and this rebuild (a late SubscribeAsync) is bound + // to the dead subscriber; it must never be reused as this runtime's pump. + EventPump? stale; + lock (_pumpLock) { stale = _eventPump; _eventPump = null; } + if (stale is not null) + { + stale.OnDataChange -= OnPumpDataChange; + try { await stale.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "Stale EventPump dispose failed before runtime rebuild"); } + _subscriptions.Clear(); + } + _ownsRuntime = true; var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false); _ownedMxClient = MxGatewayClient.Create(clientOptions); @@ -562,6 +693,7 @@ public sealed class GalaxyDriver private async Task TeardownRuntimeAsync(bool final, CancellationToken cancellationToken) { _runtimeStopping = true; + Interlocked.Increment(ref _runtimeGeneration); try { var supervisor = _supervisor; @@ -628,9 +760,16 @@ public sealed class GalaxyDriver if (_ownedMxSession is not null) { - try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); } + var session = _ownedMxSession; _ownedMxSession = null; + // Bounded close: the caller's token plus SessionCloseDeadline, so a blackholed gateway + // cannot park this (and the host's PostStop) for the SDK's 30 s call timeout. + using var closeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + closeCts.CancelAfter(SessionCloseDeadline); + try { await session.CloseAsync(closeCts.Token).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxSession close failed"); } + try { await session.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); } } if (_ownedMxClient is not null) @@ -655,6 +794,7 @@ public sealed class GalaxyDriver _dataWriter = null; _hierarchySource = null; _alarmAcknowledger = null; + _ownedPlatformSource = null; _ownsRuntime = false; } } @@ -719,7 +859,8 @@ public sealed class GalaxyDriver // completed loop. Recreate the pump so the replayed subscriptions have a consumer. RestartEventPumpForReplay(); - var tagCount = await ResubscribeAllAsync(entries, cancellationToken).ConfigureAwait(false); + var tagCount = await ResubscribeAllAsync(entries, Volatile.Read(ref _runtimeGeneration), cancellationToken) + .ConfigureAwait(false); _logger.LogInformation( "GalaxyDriver {InstanceId} replay completed — {SubCount} subscriptions, {TagCount} tags re-subscribed", @@ -733,14 +874,28 @@ public sealed class GalaxyDriver /// replay must recreate (its loop died with the stream) and a gap must NOT touch (the /// stream carrying the sentinel is still healthy and still delivering). /// + /// The subscriptions to re-issue. + /// + /// The runtime generation this re-subscribe belongs to. Re-checked before every + /// : a teardown bumps it and clears the registry, + /// and a Rebind landing after that would resurrect dead handles into the next runtime. + /// + /// Cancellation token. /// The number of tag references re-subscribed. private async Task ResubscribeAllAsync( IReadOnlyList<(long SubscriptionId, IReadOnlyList Bindings)> entries, + int generation, CancellationToken cancellationToken) { var tagCount = 0; foreach (var (subscriptionId, oldBindings) in entries) { + if (Volatile.Read(ref _runtimeGeneration) != generation) + { + _logger.LogDebug("GalaxyDriver {InstanceId} re-subscribe abandoned — runtime was torn down.", _driverInstanceId); + return tagCount; + } + if (subscriptionId == SubscriptionRegistry.ProbeSubscriptionId && _probeWatcher is { } probeWatcher) { // The probe watcher owns its handles: it re-subscribes its platforms, records the @@ -771,7 +926,9 @@ public sealed class GalaxyDriver } // Rebind so the EventPump fan-out reverse map points at the post-reconnect - // handles; otherwise events on the new handles miss every subscription. + // handles; otherwise events on the new handles miss every subscription. Not after a + // teardown: the fresh handles belong to a subscriber that is already being released. + if (Volatile.Read(ref _runtimeGeneration) != generation) return tagCount; _subscriptions.Rebind(subscriptionId, newBindings); tagCount += refs.Length; } @@ -915,8 +1072,12 @@ public sealed class GalaxyDriver if (startTask.IsFaulted) startTask.GetAwaiter().GetResult(); } - private void OnDeployWatcherRediscoveryNeeded(object? sender, RediscoveryEventArgs args) => + private void OnDeployWatcherRediscoveryNeeded(object? sender, RediscoveryEventArgs args) + { OnRediscoveryNeeded?.Invoke(this, args); + // A redeploy may add or remove platforms / engines; refresh the ScanState probe set. + ReseedProbePlatformsOnRediscovery(); + } /// public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) @@ -1437,6 +1598,15 @@ public sealed class GalaxyDriver { lock (_pumpLock) { + // Checked BEFORE the reuse: the flag is set before the teardown takes the pump under this + // same lock, so while it is set the live pump belongs to a dying subscriber. Handing it out + // would register handles into a registry the teardown is about to clear; creating a new one + // would leave a pump on the dead subscriber for the next InitializeAsync to inherit. + if (_runtimeStopping) + { + throw new InvalidOperationException( + "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync."); + } if (_eventPump is not null) return _eventPump; _eventPump = new EventPump( _subscriber!, _subscriptions, _logger, @@ -1497,6 +1667,7 @@ public sealed class GalaxyDriver return; } + var generation = Volatile.Read(ref _runtimeGeneration); _ = Task.Run(async () => { try @@ -1504,7 +1675,7 @@ public sealed class GalaxyDriver var entries = _subscriptions.SnapshotEntries(); if (entries.Count == 0) return; - var tagCount = await ResubscribeAllAsync(entries, CancellationToken.None).ConfigureAwait(false); + var tagCount = await ResubscribeAllAsync(entries, generation, CancellationToken.None).ConfigureAwait(false); _logger.LogInformation( "GalaxyDriver {InstanceId} replay-gap re-snapshot completed — {SubCount} subscriptions, {TagCount} tags re-subscribed", _driverInstanceId, entries.Count, tagCount); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs new file mode 100644 index 00000000..a385ecfb --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs @@ -0,0 +1,60 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse; + +/// The platform-only query the driver seeds its ScanState probe watcher from. +public sealed class GatewayGalaxyPlatformSourceTests +{ + private static GalaxyObject Obj(string tag, params string[] chain) + { + var o = new GalaxyObject { TagName = tag }; + o.TemplateChain.Add(chain); + return o; + } + + /// Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered. + [Fact] + public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered() + { + var requests = new List(); + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + requests.Add(req); + var reply = new DiscoverHierarchyReply(); + if (req.PageToken == "") + { + reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]); + reply.NextPageToken = "p2"; + } + else + { + reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]); + } + return Task.FromResult(reply); + } + + var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None); + + names.ShouldBe(["Plat01", "Eng01"]); + requests.Count.ShouldBe(2); + requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false); + requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine")); + } + + /// A gateway that strips the template chain is trusted to have filtered server-side. + [Fact] + public async Task Objects_without_a_template_chain_are_kept() + { + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + var reply = new DiscoverHierarchyReply(); + reply.Objects.Add(new GalaxyObject { TagName = "Plat01" }); + return Task.FromResult(reply); + } + + (await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 0, CancellationToken.None)).ShouldBe(["Plat01"]); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs new file mode 100644 index 00000000..3a7b241e --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs @@ -0,0 +1,396 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Lane F fix round 1 (review of #735): the probe watcher is seeded at InitializeAsync +/// and re-seeded on rediscovery (the runtime never calls DiscoverAsync on a live driver); +/// a SubscribeAsync landing mid-teardown cannot create a pump on the dying subscriber; a +/// gateway close that never answers cannot park ShutdownAsync; a re-subscribe that +/// straddles a teardown never rebinds into the next runtime's registry. +/// +public sealed class GalaxyDriverFixRound1Tests +{ + private const int WaitMs = 10_000; + + private static GalaxyDriverOptions Opts(bool watchDeploy = false) => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("FixRound1"), + new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), + new GalaxyReconnectOptions()); + + // ───────────────────────────── fakes ───────────────────────────── + + private sealed class ScriptedPlatformSource : IGalaxyPlatformSource + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + public Func> Script { get; init; } = _ => ["Plat01"]; + + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => Task.FromResult(Script(Interlocked.Increment(ref _calls))); + } + + private sealed class ParkableSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + private int _nextHandle; + private int _subscribeCalls; + + public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); + public List Subscribed { get; } = []; + public int SubscribeCalls => Volatile.Read(ref _subscribeCalls); + + /// When set, UnsubscribeBulk parks until . + public bool ParkUnsubscribe { get; set; } + public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult(); + + /// When > 0, that SubscribeBulk call (1-based) parks until . + public int ParkSubscribeCall { get; set; } + public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult(); + + public async Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var call = Interlocked.Increment(ref _subscribeCalls); + if (call == ParkSubscribeCall) + { + SubscribeEntered.TrySetResult(); + await _releaseSubscribe.Task.WaitAsync(cancellationToken); + } + var results = new List(); + lock (Subscribed) + { + foreach (var r in fullReferences) + { + var h = Interlocked.Increment(ref _nextHandle); + Subscribed.Add(r); + HandleByRef[r] = h; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + } + } + return results; + } + + public async Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) + { + if (ParkUnsubscribe) + { + UnsubscribeEntered.TrySetResult(); + await _releaseUnsubscribe.Task.WaitAsync(cancellationToken); + } + } + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { BoolValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + private sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource + { + public Channel Events { get; } = Channel.CreateUnbounded(); + public int Opened; + + public async IAsyncEnumerable WatchAsync( + DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref Opened); + await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent + { + Sequence = sequence, + TimeOfLastDeployPresent = true, + TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime), + }); + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } + + private static HostState? StateOf(GalaxyDriver driver, string host) => + driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; + + // ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ───────────────────────────── + + /// A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses. + [Fact] + public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync() + { + var sub = new ParkableSubscriber(); + var platforms = new ScriptedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + + await driver.InitializeAsync("{}", CancellationToken.None); + + platforms.Calls.ShouldBe(1); + sub.Subscribed.ShouldContain("Plat01.ScanState", "the probe watcher must be seeded at init"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump + await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, + "a ScanState event must surface the seeded platform on GetHostStatuses"); + } + + /// A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds. + [Fact] + public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds() + { + var sub = new ParkableSubscriber(); + var deploys = new ChannelDeployWatchSource(); + var platforms = new ScriptedPlatformSource + { + Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"], + }; + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null, + subscriber: sub, platformSource: platforms, deployWatchSource: deploys); + var raised = 0; + driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); + + await driver.InitializeAsync("{}", CancellationToken.None); + driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init"); + sub.Subscribed.ShouldBeEmpty(); + + await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); + await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed) + await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy + + await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2, + "the rediscovery must re-seed the probe set"); + platforms.Calls.ShouldBe(2); + sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true); + } + + private sealed class ParkingPlatformSource : IGalaxyPlatformSource + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + public int ParkCall { get; init; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _release.TrySetResult(); + + public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + { + var call = Interlocked.Increment(ref _calls); + if (call == ParkCall) + { + Entered.TrySetResult(); + await _release.Task.WaitAsync(cancellationToken); + return ["Plat01"]; // the pre-deploy view this pass fetched + } + return call == 1 ? [] : ["Plat01", "Eng02"]; + } + } + + /// + /// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts + /// after it — otherwise the probe set would stay on the stale view until the next deploy. + /// + [Fact] + public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass() + { + var sub = new ParkableSubscriber(); + var deploys = new ChannelDeployWatchSource(); + var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null, + subscriber: sub, platformSource: platforms, deployWatchSource: deploys); + var raised = 0; + driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); + try + { + await driver.InitializeAsync("{}", CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); + await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap + await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); + (await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task, + "the first rediscovery pass must be parked mid-fetch"); + + await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero)); + await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery"); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2, + "the final probe set must reflect the latest deploy, not the parked pass's stale view"); + } + + /// The production path (owned runtime) queries the platform source at init, with no DiscoverAsync. + [Fact] + public async Task Production_path_queries_the_platform_source_at_init() + { + var platforms = new ScriptedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms) + { + SessionOpenOverrideForTests = _ => Task.CompletedTask, + }; + + await driver.InitializeAsync("{}", CancellationToken.None); + + platforms.Calls.ShouldBe(1, "the owned runtime must seed its probe set at init"); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + // ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ───────────────────────────── + + /// + /// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before + /// the subscriber is released) is refused and leaves no pump; before the shutdown the same + /// call works (positive control). + /// + [Fact] + public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump() + { + var sub = new ParkableSubscriber(); + var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + await driver.InitializeAsync("{}", CancellationToken.None); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime"); + + sub.ParkUnsubscribe = true; + var shutdown = driver.ShutdownAsync(CancellationToken.None); + var subscribeCallsBefore = sub.SubscribeCalls; + try + { + (await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task, + "the teardown must be parked inside the probe watcher's unsubscribe"); + shutdown.IsCompleted.ShouldBeFalse(); + + // The pump is still live here (the teardown takes it after the probe watcher) — reusing it + // is exactly the race: the call must be refused, not served on the dying subscriber. + driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken"); + await Should.ThrowAsync( + () => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None)); + sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway"); + } + finally + { + // Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs. + sub.ReleaseUnsubscribe(); + } + await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown"); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + + await driver.InitializeAsync("{}", CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump"); + } + + // ───────────────────────────── IMPORTANT 3: bounded gateway close ───────────────────────────── + + /// A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline. + [Fact] + public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers() + { + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource()) + { + SessionOpenOverrideForTests = _ => Task.CompletedTask, + SessionCloseDeadline = TimeSpan.FromMilliseconds(300), + }; + await driver.InitializeAsync("{}", CancellationToken.None); + var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closeToken = CancellationToken.None; + driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct => + { + closeToken = ct; + closeEntered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway + }; + + var sw = System.Diagnostics.Stopwatch.StartNew(); + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + sw.Stop(); + + closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted"); + closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline"); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s"); + driver.OwnedMxSessionForTests.ShouldBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + } + + // ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ───────────────────────────── + + /// + /// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared + /// registry; the same replay without a teardown rebinds (positive control). + /// + [Fact] + public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind() + { + // Positive control first. + { + var sub = new ParkableSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + await driver.InvokeReplayForTestAsync(CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription"); + sub.SubscribeCalls.ShouldBe(2); + } + + { + var sub = new ParkableSubscriber { ParkSubscribeCall = 2 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + + var replay = driver.InvokeReplayForTestAsync(CancellationToken.None); + (await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task, + "the replay must be parked inside its SubscribeBulk"); + + await driver.ShutdownAsync(CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + + sub.ReleaseSubscribe(); + await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles"); + } + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs index 5a146656..10ef4ff3 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs @@ -125,6 +125,13 @@ public sealed class GalaxyDriverShutdownTests }); } + /// Canned platform set so a production-path init never dials the (fake) gateway for one. + private sealed class NoPlatforms : IGalaxyPlatformSource + { + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => Task.FromResult>([]); + } + private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed { public int Starts; @@ -172,8 +179,9 @@ public sealed class GalaxyDriverShutdownTests await driver.ShutdownAsync(CancellationToken.None); - (await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task, - "ShutdownAsync must cancel the StreamEvents stream"); + // No wait: the pump's Cancel runs the stream's registration synchronously and ShutdownAsync + // awaited the pump's dispose, so the signal is either already set or the stream was never cancelled. + sub.StreamCancelled.Task.IsCompleted.ShouldBeTrue("ShutdownAsync must cancel the StreamEvents stream"); driver.HasEventPumpForTests.ShouldBeFalse(); driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init"); driver.GetHealth().State.ShouldBe(DriverState.Unknown); @@ -289,7 +297,8 @@ public sealed class GalaxyDriverShutdownTests { var opens = 0; var source = new CountingDeployWatchSource(); - await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source) + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source, + platformSource: new NoPlatforms()) { SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; }, }; @@ -338,7 +347,8 @@ public sealed class GalaxyDriverShutdownTests public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds() { var attempts = 0; - await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null) + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, + platformSource: new NoPlatforms()) { SessionOpenOverrideForTests = _ => ++attempts == 1 ? throw new InvalidOperationException("gateway refused") From 77ca10aa37d21b03b43fe54096137f4b58e7ec78 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:29:11 -0400 Subject: [PATCH 6/8] test(galaxy): de-flake writer-lane and supervisor tests; align EffectiveTimeout doc The writer concurrency fake parks on its release signal (the 300 ms delay could release the gate early on a starved pool), the supervisor test's cross-thread hold flag is Volatile, and the EffectiveTimeout doc now states the 1 s floor the code and tests implement. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs | 4 +++- .../Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs | 4 +++- .../Runtime/ReconnectSupervisorCycleCtsTests.cs | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs index 81807500..cfa757b8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs @@ -165,7 +165,9 @@ public sealed class GalaxyDriverProbe : IDriverProbe /// config's when that is positive and /// shorter. The knob was authored by the AdminUI and parsed by the factory but consumed by /// nothing — the probe always ran on the caller's budget alone. A non-positive caller budget - /// is treated as the config value (or 1 s when that is unset too), as before. + /// is floored to 1 s before the comparison (the same floor + /// applies to the gRPC call budget), so the probe runs briefly rather than cancelling before + /// it starts; a longer config value never raises it above that floor. /// /// The budget the host handed the probe. /// The config's ProbeTimeoutSeconds; ignored when not positive. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs index dff0f665..51b4cdfb 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs @@ -51,7 +51,9 @@ public sealed class GatewayGalaxyDataWriterConcurrencyTests int seen; do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen); if (now >= ReleaseAt) _release.TrySetResult(); - await Task.WhenAny(_release.Task, Task.Delay(300, ct)); + // Park on the signal; the delay is only a safety net against a wedged test (a starved + // pool must not release the gate early and turn the peak assertion flaky). + await Task.WhenAny(_release.Task, Task.Delay(WaitMs, ct)); Interlocked.Decrement(ref _inFlight); return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle)); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs index 1660751d..4513efa6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs @@ -23,11 +23,11 @@ public sealed class ReconnectSupervisorCycleCtsTests { var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hold = false; + var hold = new[] { false }; // read on the recovery loop's thread, written here — Volatile both ways using var sup = new ReconnectSupervisor( reopen: async _ => { - if (hold) + if (Volatile.Read(ref hold[0])) { reopenEntered.TrySetResult(); await holdReopen.Task; @@ -45,7 +45,7 @@ public sealed class ReconnectSupervisorCycleCtsTests sup.CyclesStarted.ShouldBe(3); // Positive control: a cycle parked inside reopen still holds its CTS. - hold = true; + Volatile.Write(ref hold[0], true); sup.ReportTransportFailure(new IOException("drop 3")); (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task); sup.CyclesStarted.ShouldBe(4); From 3e60a925dc3dd3611d3b27ea9b7c44efe011114c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:47:34 -0400 Subject: [PATCH 7/8] fix(galaxy): background probe seed; generation-checked registrations Review fix round 2 for lane F: - N1: the init platform seed no longer runs inline under the init token. Galaxy's Tier-A Initialize budget is 5 s, and a cold gateway DiscoverHierarchy can spend all of it, so the OperationCanceledException rethrow failed init, opened the Initialize breaker and cost a full rebuild per retry. InitializeAsync now only resolves the platform source (no network) and queues a background seed tied to a per-runtime CTS that the teardown cancels. A result that lands after its runtime was torn down is discarded. - N2: the seed worker serves the runtime that is live when each pass STARTS, so a request made under a new runtime while a stale pass is in flight always gets a pass of its own. - N3: SubscribeAsync and the subscribe-once read capture the runtime generation before SubscribeBulk and register only if it is unchanged, atomically with the teardown's registry Clear (_registryGate). On a mismatch they release the handles and fail the call. Re-subscribe Rebind uses the same gate, and the production rebuild clears the registry unconditionally. - PerPlatformProbeWatcher re-checks its stop after the SubscribeBulk await, atomically with the record and publish. A pass that straddles StopAsync releases the handles it acquired instead of writing the probe entry back into a cleared registry. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../GalaxyDriver.cs | 224 +++++++++---- .../Health/PerPlatformProbeWatcher.cs | 75 ++++- .../Runtime/GalaxyDriverFixRound1Tests.cs | 26 +- .../Runtime/GalaxyDriverFixRound2Tests.cs | 313 ++++++++++++++++++ 4 files changed, 546 insertions(+), 92 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 8fc00fa8..d0fac54a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -157,10 +157,22 @@ public sealed class GalaxyDriver // from DiscoverAsync alone left the production watcher with no platforms, ever. private readonly IGalaxyPlatformSource? _platformSource; private IGalaxyPlatformSource? _ownedPlatformSource; - // Rediscovery re-seed requests not yet served. The first request starts the worker; later ones - // only bump the count, and the worker loops until it has served every request it saw — so a - // deploy that lands while a pass is mid-fetch still gets a pass that starts after it. + // Probe-seed requests (init + every rediscovery) not yet served. The first request starts the + // background worker; later ones only bump the count, and the worker loops until it has served + // every request it saw — so a request that lands while a pass is mid-fetch (a deploy, or the + // next runtime's init after a teardown) still gets a pass that starts after it. private int _reseedRequests; + private volatile string _seedReason = "initialize"; + + // Cancels the in-flight background seed of the CURRENT runtime. Created by InitializeAsync, + // cancelled (never disposed — it has no timer or wait handle, and the worker may still read its + // token) by the teardown. + private CancellationTokenSource? _seedCts; + + // Serialises "is this registration still for the live runtime?" against the teardown's registry + // Clear, so a subscribe / re-subscribe whose SubscribeBulk straddled a teardown either lands + // before the Clear (and is wiped by it) or sees the bumped generation and backs out. + private readonly Lock _registryGate = new(); // Bumped by every runtime teardown. Background work started under one runtime (the replay-gap // re-snapshot, a rediscovery re-seed) re-checks it before touching the registry / watcher, so a @@ -475,7 +487,11 @@ public sealed class GalaxyDriver } await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false); - await SeedProbePlatformsAsync("initialize", cancellationToken).ConfigureAwait(false); + // The platform seed is a gateway DiscoverHierarchy round-trip, which can spend the whole of + // a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It + // runs in the background: InitializeAsync neither waits on it nor fails because of it. + await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false); + RequestProbeSeed("initialize"); _logger.LogInformation( "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName); @@ -488,39 +504,95 @@ public sealed class GalaxyDriver } /// - /// Fetch the galaxy's platform / engine set and hand it to the probe watcher. Best-effort: - /// a failed fetch logs a Warning and leaves the watcher as it was — it must never fail an - /// InitializeAsync (the value path does not depend on it), and the next rediscovery - /// re-seeds. No-op when there is no watcher or no platform source (seam mode without one). + /// Under the lifecycle gate, before the first seed request of a runtime: resolve the platform + /// source (production builds it over the owned repository client — no network here) and arm + /// the runtime's seed cancellation. Resolving here, not in the worker, keeps the background + /// pass from creating a repository client after a teardown released it. A failure is a + /// Warning: the value path does not depend on the probe set. /// - private async Task SeedProbePlatformsAsync(string reason, CancellationToken cancellationToken) + private async Task PrepareProbeSeedAsync(CancellationToken cancellationToken) { - var watcher = _probeWatcher; - if (watcher is null) return; - - IGalaxyPlatformSource? source; + if (_probeWatcher is null) return; + _seedCts ??= new CancellationTokenSource(); + if (_platformSource is not null || !_ownsRuntime || _ownedPlatformSource is not null) return; try { - source = await ResolvePlatformSourceAsync(cancellationToken).ConfigureAwait(false); + _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( + await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + _ownedPlatformSource = + new GatewayGalaxyPlatformSource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize); } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) { _logger.LogWarning(ex, - "GalaxyDriver {InstanceId} could not build the platform query ({Reason}) — per-platform host status stays unseeded until the next rediscovery.", - _driverInstanceId, reason); - return; + "GalaxyDriver {InstanceId} could not build the platform query — per-platform host status stays unseeded until the next rediscovery.", + _driverInstanceId); } - if (source is null) return; + } + + /// + /// Ask for a background probe seed (init and every deploy-watcher rediscovery). Requests + /// coalesce onto one worker; each worker pass serves the runtime that is live when the pass + /// STARTS, and a pass whose fetch completes after a teardown is discarded — so a request made + /// under a new runtime always gets a pass of its own. + /// + private void RequestProbeSeed(string reason) + { + if (_runtimeStopping || _probeWatcher is null) return; + _seedReason = reason; + if (Interlocked.Increment(ref _reseedRequests) != 1) return; // the running worker will serve it + + _ = Task.Run(async () => + { + int served; + do + { + served = Volatile.Read(ref _reseedRequests); + var generation = Volatile.Read(ref _runtimeGeneration); + var seedCts = Volatile.Read(ref _seedCts); + if (!_runtimeStopping && seedCts is not null) + { + await SeedProbePlatformsAsync(_seedReason, generation, seedCts.Token).ConfigureAwait(false); + } + } + while (Interlocked.Add(ref _reseedRequests, -served) != 0); + }); + } + + /// + /// One seed pass: fetch the platform / engine set and hand it to the probe watcher of the + /// runtime identified by . Never throws — a failed or cancelled + /// fetch logs and leaves the watcher as it was (the next rediscovery retries), and a result + /// that lands after that runtime was torn down is discarded. + /// + private async Task SeedProbePlatformsAsync(string reason, int generation, CancellationToken cancellationToken) + { + var watcher = _probeWatcher; + var source = _platformSource ?? _ownedPlatformSource; + if (watcher is null || source is null) return; try { var platforms = await source.GetPlatformTagNamesAsync(cancellationToken).ConfigureAwait(false); + if (Volatile.Read(ref _runtimeGeneration) != generation || _runtimeStopping + || !ReferenceEquals(watcher, _probeWatcher)) + { + _logger.LogDebug( + "GalaxyDriver {InstanceId} discarded a platform seed ({Reason}) — its runtime was torn down.", + _driverInstanceId, reason); + return; + } await watcher.SyncPlatformsAsync(platforms, cancellationToken).ConfigureAwait(false); _logger.LogInformation( "GalaxyDriver {InstanceId} probe watcher seeded with {Count} platform(s) ({Reason}).", _driverInstanceId, platforms.Count, reason); } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (Exception ex) when (cancellationToken.IsCancellationRequested + || Volatile.Read(ref _runtimeGeneration) != generation) + { + _logger.LogDebug(ex, "GalaxyDriver {InstanceId} platform seed ({Reason}) abandoned by a teardown.", + _driverInstanceId, reason); + } catch (Exception ex) { _logger.LogWarning(ex, @@ -529,50 +601,48 @@ public sealed class GalaxyDriver } } - private async Task ResolvePlatformSourceAsync(CancellationToken cancellationToken) - { - if (_platformSource is not null) return _platformSource; - if (!_ownsRuntime) return null; // seam mode without an injected source: nothing to query - if (_ownedPlatformSource is not null) return _ownedPlatformSource; + /// Deploy-watcher rediscovery: a redeploy may add or remove platforms / engines. + private void ReseedProbePlatformsOnRediscovery() => RequestProbeSeed("rediscovery"); - _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( - await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); - return _ownedPlatformSource = - new GatewayGalaxyPlatformSource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize); + /// + /// Register only while is still the + /// live runtime. Atomic with the teardown's registry Clear (both under ). + /// + private bool TryRegisterForGeneration(long subscriptionId, IReadOnlyList bindings, int generation) + { + lock (_registryGate) + { + if (Volatile.Read(ref _runtimeGeneration) != generation) return false; + _subscriptions.Register(subscriptionId, bindings); + return true; + } + } + + private void ClearRegistry() + { + lock (_registryGate) _subscriptions.Clear(); } /// - /// Deploy-watcher rediscovery: re-seed the platform set off the watcher's thread, one pass at - /// a time, and only if the runtime it was raised under is still the live one. + /// Best-effort release of the live handles a SubscribeBulk produced for a call that then lost + /// its runtime — the subscriber may already be dead, so a failure is logged and swallowed. /// - private void ReseedProbePlatformsOnRediscovery() + private async Task ReleaseOrphanedHandlesAsync(IGalaxySubscriber subscriber, IReadOnlyList bindings) { - if (_runtimeStopping || _probeWatcher is null) return; - if (Interlocked.Increment(ref _reseedRequests) != 1) return; // the running worker will serve it - var generation = Volatile.Read(ref _runtimeGeneration); - - _ = Task.Run(async () => + var live = bindings.Where(b => b.ItemHandle > 0).Select(b => b.ItemHandle).ToArray(); + if (live.Length == 0) return; + try { await subscriber.UnsubscribeBulkAsync(live, CancellationToken.None).ConfigureAwait(false); } + catch (Exception ex) { - int served; - do - { - served = Volatile.Read(ref _reseedRequests); - try - { - if (Volatile.Read(ref _runtimeGeneration) == generation && !_runtimeStopping) - { - await SeedProbePlatformsAsync("rediscovery", CancellationToken.None).ConfigureAwait(false); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "GalaxyDriver {InstanceId} rediscovery re-seed failed.", _driverInstanceId); - } - } - while (Interlocked.Add(ref _reseedRequests, -served) != 0); - }); + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} could not release {Count} handle(s) orphaned by a teardown.", + _driverInstanceId, live.Length); + } } + private static InvalidOperationException TornDownDuringSubscribe() => new( + "GalaxyDriver runtime was torn down while the subscribe was in flight — retry after the next InitializeAsync."); + // The watcher publishes its probe handles into the driver's registry (reserved id) so the // EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every // probe event was dropped before it got anywhere near the watcher. @@ -616,8 +686,9 @@ public sealed class GalaxyDriver stale.OnDataChange -= OnPumpDataChange; try { await stale.DisposeAsync().ConfigureAwait(false); } catch (Exception ex) { _logger.LogWarning(ex, "Stale EventPump dispose failed before runtime rebuild"); } - _subscriptions.Clear(); } + // Unconditionally: whatever the registry holds names handles of a session this rebuild replaces. + ClearRegistry(); _ownsRuntime = true; var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false); @@ -694,6 +765,7 @@ public sealed class GalaxyDriver { _runtimeStopping = true; Interlocked.Increment(ref _runtimeGeneration); + Interlocked.Exchange(ref _seedCts, null)?.Cancel(); // a seed in flight belongs to this runtime try { var supervisor = _supervisor; @@ -735,7 +807,7 @@ public sealed class GalaxyDriver } // Every item handle the registry knows died with the pump's session; the host // re-subscribes (fresh ids) once the next InitializeAsync reports Connected. - _subscriptions.Clear(); + ClearRegistry(); // The alarm feed: owned → disposed and nulled so InitializeAsync builds a fresh one; // injected → detached (the injector owns its lifetime) unless this is the final dispose. @@ -928,8 +1000,11 @@ public sealed class GalaxyDriver // Rebind so the EventPump fan-out reverse map points at the post-reconnect // handles; otherwise events on the new handles miss every subscription. Not after a // teardown: the fresh handles belong to a subscriber that is already being released. - if (Volatile.Read(ref _runtimeGeneration) != generation) return tagCount; - _subscriptions.Rebind(subscriptionId, newBindings); + lock (_registryGate) + { + if (Volatile.Read(ref _runtimeGeneration) != generation) return tagCount; + _subscriptions.Rebind(subscriptionId, newBindings); + } tagCount += refs.Length; } @@ -1308,6 +1383,8 @@ public sealed class GalaxyDriver deadlineCts.CancelAfter(ReadViaSubscribeDeadline); var cancellationToken = deadlineCts.Token; + var generation = Volatile.Read(ref _runtimeGeneration); // see SubscribeAsync + var readSubscriber = _subscriber!; var pump = EnsureEventPumpStarted(); var subscriptionId = _subscriptions.NextSubscriptionId(); @@ -1341,7 +1418,7 @@ public sealed class GalaxyDriver // The RPC gets only the CALLER's token: the internal deadline bounds our wait, not the RPC, so a slow // gateway can never surface an OperationCanceledException to a caller that did not cancel, and the // handles it eventually creates are known (and unsubscribed below) rather than lost to a cancelled call. - var subscribeTask = _subscriber!.SubscribeBulkAsync(fullReferences, bufferedIntervalMs, callerToken); + var subscribeTask = readSubscriber.SubscribeBulkAsync(fullReferences, bufferedIntervalMs, callerToken); try { results = await subscribeTask.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -1350,7 +1427,7 @@ public sealed class GalaxyDriver { // Internal deadline fired while the subscribe RPC was still in flight. pump.OnDataChange -= handler; - var subscriber = _subscriber!; + var subscriber = readSubscriber; var logger = _logger; _ = subscribeTask.ContinueWith(t => { @@ -1402,7 +1479,12 @@ public sealed class GalaxyDriver ServerTimestampUtc: DateTime.UtcNow)); } } - _subscriptions.Register(subscriptionId, bindings); + if (!TryRegisterForGeneration(subscriptionId, bindings, generation)) + { + pump.OnDataChange -= handler; + await ReleaseOrphanedHandlesAsync(readSubscriber, bindings).ConfigureAwait(false); + throw TornDownDuringSubscribe(); + } try { @@ -1441,7 +1523,7 @@ public sealed class GalaxyDriver { try { - await _subscriber!.UnsubscribeBulkAsync(liveHandles, CancellationToken.None) + await readSubscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None) .ConfigureAwait(false); } catch (Exception ex) @@ -1502,6 +1584,10 @@ public sealed class GalaxyDriver "Call InitializeAsync and wait for a successful connection before subscribing."); } + // Captured before the pump/stopping check: a teardown that starts after this point bumps the + // generation, so the Register below can tell its SubscribeBulk straddled a teardown. + var generation = Volatile.Read(ref _runtimeGeneration); + var subscriber = _subscriber; var pump = EnsureEventPumpStarted(); var subscriptionId = _subscriptions.NextSubscriptionId(); @@ -1509,7 +1595,7 @@ public sealed class GalaxyDriver { // Empty subscriptions register but never bind anything — keeps Unsubscribe // symmetric for callers that conditionally add tags later. - _subscriptions.Register(subscriptionId, []); + if (!TryRegisterForGeneration(subscriptionId, [], generation)) throw TornDownDuringSubscribe(); return new GalaxySubscriptionHandle(subscriptionId); } @@ -1525,7 +1611,7 @@ public sealed class GalaxyDriver // and infrastructure callers (probe watcher, deploy watcher) hit the fallback. var requested = (int)Math.Max(0, publishingInterval.TotalMilliseconds); var bufferedIntervalMs = requested > 0 ? requested : _options.MxAccess.PublishingIntervalMs; - var results = await _subscriber + var results = await subscriber .SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken) .ConfigureAwait(false); @@ -1553,7 +1639,13 @@ public sealed class GalaxyDriver } } - _subscriptions.Register(subscriptionId, bindings); + if (!TryRegisterForGeneration(subscriptionId, bindings, generation)) + { + // The teardown cleared the registry while SubscribeBulk was in flight; registering now + // would carry dead handles into the next runtime. Release what we got and fail the call. + await ReleaseOrphanedHandlesAsync(subscriber, bindings).ConfigureAwait(false); + throw TornDownDuringSubscribe(); + } _ = pump; // keep the pump alive for the subscription's lifetime return new GalaxySubscriptionHandle(subscriptionId); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs index ae4c41ac..64080d52 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs @@ -49,7 +49,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable private readonly ConcurrentDictionary _itemHandlesByPlatform = new(StringComparer.OrdinalIgnoreCase); private readonly Lock _syncLock = new(); - private bool _disposed; + // Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop. + private volatile bool _disposed; /// Initializes a new instance of the PerPlatformProbeWatcher class. /// The Galaxy subscriber for managing probe subscriptions. @@ -100,10 +101,18 @@ public sealed class PerPlatformProbeWatcher : IDisposable private IReadOnlyList BuildBindings() => [.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))]; - /// Mirror the tracked handle set into the driver registry (no-op without one). + /// + /// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped — + /// a stopped watcher must never write its entry back into a registry the teardown cleared). + /// private void PublishBindings() { - if (_registry is null) return; + lock (_syncLock) PublishBindingsLocked(); + } + + private void PublishBindingsLocked() + { + if (_registry is null || _disposed) return; var bindings = BuildBindings(); if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId); else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings); @@ -136,24 +145,51 @@ public sealed class PerPlatformProbeWatcher : IDisposable var results = await _subscriber.SubscribeBulkAsync( probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false); + var acquired = new (string Platform, int ItemHandle, string? Error)[platforms.Count]; for (var i = 0; i < platforms.Count; i++) { - var platform = platforms[i]; var match = results.FirstOrDefault(r => string.Equals( r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase)); + acquired[i] = (platforms[i], match is { WasSuccessful: true } ? match.ItemHandle : 0, + match?.ErrorMessage ?? ""); + } - var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0; - _itemHandlesByPlatform[platform] = itemHandle; - - if (itemHandle <= 0) + // Re-check the stop AFTER the await, atomically with the record + publish: a StopAsync that + // ran while SubscribeBulk was in flight already cleared the map and the registry entry, so + // recording now would resurrect the probe entry into the next runtime's registry. + bool stopped; + lock (_syncLock) + { + stopped = _disposed; + if (!stopped) { - _logger.LogWarning( - "PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", - platform, match?.ErrorMessage ?? ""); + foreach (var (platform, itemHandle, _) in acquired) _itemHandlesByPlatform[platform] = itemHandle; + PublishBindingsLocked(); } } - PublishBindings(); + if (stopped) + { + var orphans = acquired.Where(a => a.ItemHandle > 0).Select(a => a.ItemHandle).ToArray(); + if (orphans.Length > 0) + { + try { await _subscriber.UnsubscribeBulkAsync(orphans, CancellationToken.None).ConfigureAwait(false); } + catch (Exception ex) + { + _logger.LogWarning(ex, + "PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length); + } + } + return; + } + + foreach (var (platform, itemHandle, error) in acquired) + { + if (itemHandle <= 0) + { + _logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error); + } + } } /// Snapshot of platform tag names currently watched. @@ -274,12 +310,15 @@ public sealed class PerPlatformProbeWatcher : IDisposable /// A task that completes once the handles have been released (or the attempt gave up). public async Task StopAsync(CancellationToken cancellationToken) { - if (_disposed) return; - _disposed = true; - - var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray(); - _itemHandlesByPlatform.Clear(); - _registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId); + int[] liveHandles; + lock (_syncLock) + { + if (_disposed) return; + _disposed = true; + liveHandles = [.. _itemHandlesByPlatform.Values.Where(h => h > 0)]; + _itemHandlesByPlatform.Clear(); + _registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId); + } if (liveHandles.Length > 0) { try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs index 3a7b241e..9876c09f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs @@ -21,9 +21,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// public sealed class GalaxyDriverFixRound1Tests { - private const int WaitMs = 10_000; + internal const int WaitMs = 10_000; - private static GalaxyDriverOptions Opts(bool watchDeploy = false) => new( + internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new( new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), new GalaxyMxAccessOptions("FixRound1"), new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), @@ -41,7 +41,7 @@ public sealed class GalaxyDriverFixRound1Tests => Task.FromResult(Script(Interlocked.Increment(ref _calls))); } - private sealed class ParkableSubscriber : IGalaxySubscriber + internal sealed class ParkableSubscriber : IGalaxySubscriber { private readonly Channel _events = Channel.CreateUnbounded(); private int _nextHandle; @@ -86,8 +86,12 @@ public sealed class GalaxyDriverFixRound1Tests return results; } + /// Every handle passed to UnsubscribeBulk, in call order. + public List Unsubscribed { get; } = []; + public async Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) { + lock (Unsubscribed) Unsubscribed.AddRange(itemHandles); if (ParkUnsubscribe) { UnsubscribeEntered.TrySetResult(); @@ -113,7 +117,7 @@ public sealed class GalaxyDriverFixRound1Tests }); } - private sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource + internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource { public Channel Events { get; } = Channel.CreateUnbounded(); public int Opened; @@ -136,7 +140,7 @@ public sealed class GalaxyDriverFixRound1Tests }); } - private static async Task WaitUntilAsync(Func condition, string because) + internal static async Task WaitUntilAsync(Func condition, string because) { var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); while (!condition()) @@ -146,7 +150,7 @@ public sealed class GalaxyDriverFixRound1Tests } } - private static HostState? StateOf(GalaxyDriver driver, string host) => + internal static HostState? StateOf(GalaxyDriver driver, string host) => driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; // ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ───────────────────────────── @@ -161,8 +165,11 @@ public sealed class GalaxyDriverFixRound1Tests await driver.InitializeAsync("{}", CancellationToken.None); + // The seed runs in the background (fix round 2, N1) — init does not wait on it. + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the probe watcher must be seeded after init"); platforms.Calls.ShouldBe(1); - sub.Subscribed.ShouldContain("Plat01.ScanState", "the probe watcher must be seeded at init"); + lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState"); driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump @@ -188,6 +195,7 @@ public sealed class GalaxyDriverFixRound1Tests await driver.InitializeAsync("{}", CancellationToken.None); driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init"); + await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run"); sub.Subscribed.ShouldBeEmpty(); await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); @@ -271,7 +279,7 @@ public sealed class GalaxyDriverFixRound1Tests await driver.InitializeAsync("{}", CancellationToken.None); - platforms.Calls.ShouldBe(1, "the owned runtime must seed its probe set at init"); + await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init"); driver.GetHealth().State.ShouldBe(DriverState.Healthy); } @@ -289,6 +297,8 @@ public sealed class GalaxyDriverFixRound1Tests var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); await driver.InitializeAsync("{}", CancellationToken.None); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the (background) seed must give the probe watcher its handle first"); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime"); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs new file mode 100644 index 00000000..ddd97023 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs @@ -0,0 +1,313 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; +using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Lane F fix round 2 (re-review of #735): +/// +/// N1 — the init platform seed runs in the background: a slow or cancelled repository +/// fetch can neither delay nor fail InitializeAsync (Galaxy's Tier-A Initialize budget is +/// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown +/// is discarded. +/// N2 — a seed requested under a new runtime always gets a pass of its own, even when a +/// worker from the previous runtime is still running. +/// N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails +/// and unsubscribes instead of registering into the next runtime's registry. +/// The probe watcher's own subscribe re-checks its stop after the await. +/// +/// +public sealed class GalaxyDriverFixRound2Tests +{ + private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2); + + /// A platform source whose fetches park until released (optionally ignoring cancellation). + private sealed class GatedPlatformSource : IGalaxyPlatformSource + { + private int _calls; + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int Calls => Volatile.Read(ref _calls); + public int ParkCall { get; init; } = 1; + public bool IgnoreCancellation { get; init; } + public Func> Result { get; init; } = _ => ["Plat01"]; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _release.TrySetResult(); + + public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + { + var call = Interlocked.Increment(ref _calls); + if (call == ParkCall) + { + ParkedToken.TrySetResult(cancellationToken); + Entered.TrySetResult(); + if (IgnoreCancellation) await _release.Task; + else await _release.Task.WaitAsync(cancellationToken); + } + return Result(call); + } + } + + private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _calls); + return Task.FromException>(ex); + } + } + + private static async Task Entered(TaskCompletionSource tcs, string because) => + (await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because); + + // ───────────────────────────── N1: background init seed ───────────────────────────── + + /// + /// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy + /// inside its budget, and the teardown cancels the parked fetch. + /// + [Fact] + public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + try + { + var sw = System.Diagnostics.Stopwatch.StartNew(); + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + sw.Stop(); + + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + await Entered(platforms.Entered, "positive control: the seed fetch was started"); + sw.Elapsed.ShouldBeLessThan(InitBudget); + + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + var token = await platforms.ParkedToken.Task; + token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight"); + } + finally + { + platforms.Release(); + } + } + + /// A fetch that is cancelled (the gateway's own deadline) does not fail init. + [Fact] + public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync() + { + var sub = new ParkableSubscriber(); + var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline")); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty(); + + // The driver keeps serving after the failed seed. + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(1); + } + + /// + /// The production failure mode: the host's Tier-A Initialize timeout cancels the init token + /// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and + /// the seed — tied to the runtime, not to the init call — must survive that cancellation and + /// still land. + /// + [Fact] + public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + using var initCts = new CancellationTokenSource(); + try + { + initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk + await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + + await Entered(platforms.Entered, "the seed fetch must be in flight"); + await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked"); + (await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse( + "the seed belongs to the runtime; the init call's timeout must not cancel it"); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the seed must still land after the init token was cancelled"); + } + + /// Positive control: a fetch that completes after init still populates host statuses. + [Fact] + public async Task A_fetch_completing_after_init_populates_host_statuses() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + try + { + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + await Entered(platforms.Entered, "the seed must be parked"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty(); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the late seed must reach the probe watcher"); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump + await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, + "a ScanState event must surface the late-seeded platform"); + } + + // ───────────────────────────── N1 generation + N2: straddling worker ───────────────────────────── + + /// + /// A seed from the previous runtime that completes after a teardown + re-init is discarded, + /// and the new runtime's own seed request still gets its own pass. + /// + [Fact] + public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource + { + IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it + Result = call => call == 1 ? ["Stale01"] : ["Plat01"], + }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + try + { + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + await Entered(platforms.Entered, "runtime 1's seed must be parked"); + + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the new runtime must be seeded"); + driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]); + lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded"); + } + + // ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ───────────────────────────── + + /// + /// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry + /// back into the registry, and must release the handles it just acquired. Positive control: + /// the same sync without a stop registers. + /// + [Fact] + public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry() + { + { + var registry = new SubscriptionRegistry(); + using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry); + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control"); + } + + { + var registry = new SubscriptionRegistry(); + var sub = new ParkableSubscriber { ParkSubscribeCall = 1 }; + var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry); + + var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk"); + await watcher.StopAsync(CancellationToken.None); + + sub.ReleaseSubscribe(); + await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register"); + watcher.WatchedPlatforms.ShouldBeEmpty(); + lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"], + "the handle acquired after the stop must be released"); + } + } + + // ───────────────────────────── N3: subscribe / read straddling a teardown ───────────────────────────── + + /// + /// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing + /// and unsubscribes its handles; a subscribe on the live runtime registers (positive control). + /// + [Fact] + public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing() + { + var sub = new ParkableSubscriber { ParkSubscribeCall = 2 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers"); + + var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None); + try + { + await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk"); + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + } + finally + { + sub.ReleaseSubscribe(); + } + + await Should.ThrowAsync(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs))); + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry"); + lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]); + + await driver.InitializeAsync("{}", CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry"); + } + + /// The subscribe-once read path: same straddle, same outcome. + [Fact] + public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing() + { + var sub = new ParkableSubscriber { ParkSubscribeCall = 1 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None); + try + { + await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk"); + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + } + finally + { + sub.ReleaseSubscribe(); + } + + await Should.ThrowAsync(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs))); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]); + } +} From 0d367590dbae5c60697d4250945f4ff730163794 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:47:34 -0400 Subject: [PATCH 8/8] test(galaxy): isolate process-wide telemetry tests; bounded stream-cancel wait GalaxyTelemetryTests (ActivityListener) and GatewayGalaxyDataWriterFailClosedTests (MeterListener, filtered only by meter and instrument name) observe every Galaxy span or measurement in the process. They now run in a collection with parallelization disabled (F-FU6), because a parallel driver test, or its background probe seed, polluted their counts. The stream-cancel assertion in GalaxyDriverShutdownTests goes back to a bounded wait: the instantaneous IsCompleted check introduced in round 1 was racy. Refs #735 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ProcessWideTelemetryCollection.cs | 17 +++++++++++++++++ .../Runtime/GalaxyDriverShutdownTests.cs | 7 ++++--- .../Runtime/GalaxyTelemetryTests.cs | 1 + .../GatewayGalaxyDataWriterFailClosedTests.cs | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs new file mode 100644 index 00000000..9d2f16fe --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs @@ -0,0 +1,17 @@ +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests; + +/// +/// Classes that assert on process-wide telemetry (an ActivityListener on the Galaxy +/// ActivitySource, a MeterListener on the Galaxy meter) run with parallelization disabled: +/// those listeners see every Galaxy span / measurement in the process, so any parallel class that +/// drives a traced subscriber or the data writer — including a driver's background probe seed that +/// outlives its test — pollutes their counts (F-FU6, observed flaky in lane F fix round 2). +/// +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class ProcessWideTelemetryCollection +{ + /// The collection name. + public const string Name = "Process-wide Galaxy telemetry"; +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs index 10ef4ff3..cdb2ec69 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs @@ -179,9 +179,10 @@ public sealed class GalaxyDriverShutdownTests await driver.ShutdownAsync(CancellationToken.None); - // No wait: the pump's Cancel runs the stream's registration synchronously and ShutdownAsync - // awaited the pump's dispose, so the signal is either already set or the stream was never cancelled. - sub.StreamCancelled.Task.IsCompleted.ShouldBeTrue("ShutdownAsync must cancel the StreamEvents stream"); + // Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the + // enumerator's cancellation registration has run by the time it returns (observed flaky). + (await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task, + "ShutdownAsync must cancel the StreamEvents stream"); driver.HasEventPumpForTests.ShouldBeFalse(); driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init"); driver.GetHealth().State.ShouldBe(DriverState.Unknown); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs index bfef6df2..6b311f9f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs @@ -14,6 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// rather than asserting on internal state, so the /// tests double as documentation of the listener-side contract. /// +[Collection(ProcessWideTelemetryCollection.Name)] public sealed class GalaxyTelemetryTests { /// Subscribes an ActivityListener for the test, captures each spawned activity. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs index 67a01d77..83d41dea 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs @@ -13,6 +13,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal /// seam (the SDK session types are sealed + internal-ctor and cannot be faked). /// +[Collection(ProcessWideTelemetryCollection.Name)] public sealed class GatewayGalaxyDataWriterFailClosedTests { private static GatewayGalaxyDataWriter NewWriter()