diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs
index 6242351c..6e3bc5f8 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs
@@ -61,8 +61,12 @@ public sealed class GalaxyDiscoverer
///
/// The address space builder to populate with discovery results.
/// The cancellation token for the operation.
- /// A task that represents the asynchronous operation.
- public async Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken)
+ ///
+ /// The hierarchy this pass fetched, so a caller that needs the object set for something
+ /// else (the driver selects its $WinPlatform / $AppEngine probe targets from
+ /// it) reuses the one sweep instead of asking the gateway for the full galaxy twice.
+ ///
+ public async Task> DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(builder);
var objects = await _source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false);
@@ -132,6 +136,8 @@ public sealed class GalaxyDiscoverer
}
}
}
+
+ return objects;
}
// Resolve (and memoise) the folder for a gobject, recursively creating its parent's folder
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs
index bafabb59..68899bb8 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs
@@ -5,22 +5,83 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
///
/// Default wrapping the gateway's
-/// . Pages internally via the client's overload.
+/// . Pages the DiscoverHierarchy RPC itself,
+/// at the configured Repository.DiscoverPageSize: the client's typed
+/// DiscoverHierarchyAsync overloads hard-code a 5000-row page
+/// (DiscoverHierarchyOptions carries no page size), so the knob the AdminUI
+/// authors and the factory parses used to reach nothing. The raw RPC takes the page
+/// size on the wire (DiscoverHierarchyRequest.page_size), so it is honoured here.
///
public sealed class GatewayGalaxyHierarchySource : IGalaxyHierarchySource
{
+ /// The page size used when the configured one is not positive — the client library's own default.
+ public const int DefaultPageSize = 5000;
+
private readonly GalaxyRepositoryClient _client;
+ private readonly int _pageSize;
///
/// Initializes a new instance of the class.
///
/// The gateway's Galaxy repository client for discovering the object hierarchy.
- public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client)
+ ///
+ /// Objects per DiscoverHierarchy page (Repository.DiscoverPageSize). A value that is
+ /// not positive falls back to .
+ ///
+ public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client, int pageSize = DefaultPageSize)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
+ _pageSize = EffectivePageSize(pageSize);
}
+ /// The page size this source sends on every DiscoverHierarchy request.
+ public int PageSize => _pageSize;
+
+ /// Clamp a configured page size to something the gateway accepts: non-positive → .
+ /// The configured value.
+ /// The page size to send.
+ public static int EffectivePageSize(int configured) => configured > 0 ? configured : DefaultPageSize;
+
///
public Task> GetHierarchyAsync(CancellationToken cancellationToken)
- => _client.DiscoverHierarchyAsync(cancellationToken);
+ => PageAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
+
+ ///
+ /// Walk every DiscoverHierarchy page at and concatenate the
+ /// objects. Mirrors the client library's own loop (including its repeated-page-token guard,
+ /// which turns a misbehaving gateway into an error rather than an endless walk), with the
+ /// RPC behind a delegate so the loop is unit-testable without a channel.
+ ///
+ /// The raw DiscoverHierarchy RPC.
+ /// Objects per page; sent on every request.
+ /// Cancellation token for the walk.
+ /// Every object across all pages, in gateway order.
+ internal static async Task> PageAsync(
+ Func> discover,
+ int pageSize,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(discover);
+ var effective = EffectivePageSize(pageSize);
+
+ var objects = new List();
+ var seenPageTokens = new HashSet(StringComparer.Ordinal);
+ var pageToken = string.Empty;
+ do
+ {
+ var request = new DiscoverHierarchyRequest { PageSize = effective, PageToken = pageToken };
+ var reply = await discover(request, cancellationToken).ConfigureAwait(false);
+
+ objects.AddRange(reply.Objects);
+ pageToken = reply.NextPageToken ?? string.Empty;
+ if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
+ {
+ throw new InvalidOperationException(
+ $"Galaxy DiscoverHierarchy returned a repeated page token '{pageToken}' — refusing to loop.");
+ }
+ }
+ while (!string.IsNullOrWhiteSpace(pageToken));
+
+ return objects;
+ }
}
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs
new file mode 100644
index 00000000..dfed1659
--- /dev/null
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs
@@ -0,0 +1,88 @@
+using ZB.MOM.WW.MxGateway.Client;
+using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+
+///
+/// Default : a platform-only DiscoverHierarchy sweep
+/// over the gateway's — the request filters on
+/// template_chain_contains ($WinPlatform, $AppEngine) and asks for no
+/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the
+/// address-space discovery pulls. Paged at the configured Repository.DiscoverPageSize.
+///
+public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
+{
+ /// Template-chain tokens that identify a platform / engine object.
+ public static readonly IReadOnlyList PlatformTemplates = ["$WinPlatform", "$AppEngine"];
+
+ private readonly GalaxyRepositoryClient _client;
+ private readonly int _pageSize;
+
+ /// Initializes a new instance of the class.
+ /// The gateway's Galaxy repository client.
+ /// Objects per page (Repository.DiscoverPageSize); non-positive → the client default.
+ public GatewayGalaxyPlatformSource(GalaxyRepositoryClient client, int pageSize = GatewayGalaxyHierarchySource.DefaultPageSize)
+ {
+ _client = client ?? throw new ArgumentNullException(nameof(client));
+ _pageSize = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize);
+ }
+
+ ///
+ public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+ => QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
+
+ ///
+ /// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every
+ /// request carries the template filter and include_attributes = false; the reply is
+ /// re-filtered on the returned template chain (a gateway that ignores the filter still
+ /// yields only platforms) and reduced to non-empty tag names.
+ ///
+ /// The raw DiscoverHierarchy RPC.
+ /// Objects per page.
+ /// Cancellation token for the walk.
+ /// Distinct platform tag names in gateway order.
+ internal static async Task> QueryAsync(
+ Func> discover,
+ int pageSize,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(discover);
+ var effective = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize);
+
+ var names = new List();
+ var seen = new HashSet(StringComparer.OrdinalIgnoreCase);
+ var seenPageTokens = new HashSet(StringComparer.Ordinal);
+ var pageToken = string.Empty;
+ do
+ {
+ var request = new DiscoverHierarchyRequest
+ {
+ PageSize = effective,
+ PageToken = pageToken,
+ IncludeAttributes = false,
+ };
+ request.TemplateChainContains.Add(PlatformTemplates);
+
+ var reply = await discover(request, cancellationToken).ConfigureAwait(false);
+ foreach (var obj in reply.Objects)
+ {
+ if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
+ if (seen.Add(obj.TagName)) names.Add(obj.TagName);
+ }
+
+ pageToken = reply.NextPageToken ?? string.Empty;
+ if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
+ {
+ throw new InvalidOperationException(
+ $"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop.");
+ }
+ }
+ while (!string.IsNullOrWhiteSpace(pageToken));
+
+ return names;
+ }
+
+ private static bool IsPlatform(GalaxyObject obj) =>
+ obj.TemplateChain.Count == 0 // a gateway that strips the chain already filtered server-side
+ || obj.TemplateChain.Any(t => PlatformTemplates.Contains(t, StringComparer.OrdinalIgnoreCase));
+}
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs
new file mode 100644
index 00000000..6589c9aa
--- /dev/null
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs
@@ -0,0 +1,18 @@
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+
+///
+/// Driver-side seam for the set of Galaxy platform / engine objects whose ScanState
+/// the PerPlatformProbeWatcher advises. Production wraps the gateway's
+/// DiscoverHierarchy RPC filtered to $WinPlatform / $AppEngine with no
+/// attributes (); tests substitute a canned list.
+/// The driver seeds the watcher from this at InitializeAsync and re-seeds on every
+/// deploy-watcher rediscovery — the runtime never calls DiscoverAsync on a live driver,
+/// so without this seed the probe watcher never had a platform set in production.
+///
+public interface IGalaxyPlatformSource
+{
+ /// The tag_name of every deployed $WinPlatform / $AppEngine object.
+ /// Cancellation token for the query.
+ /// Platform tag names; empty when the galaxy has none.
+ Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken);
+}
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs
index 3216e136..d0fac54a 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs
@@ -146,9 +146,98 @@ public sealed class GalaxyDriver
private HostConnectivityForwarder? _transportForwarder;
private PerPlatformProbeWatcher? _probeWatcher;
+ // Injected deploy-watch seam (tests). Null in production, where StartDeployWatcherAsync builds a
+ // GatewayGalaxyDeployWatchSource over the owned GalaxyRepositoryClient.
+ private readonly IGalaxyDeployWatchSource? _deployWatchSource;
+
+ // Platform set for the ScanState probe watcher. Injected seam (tests) or, on the production
+ // path, a platform-only DiscoverHierarchy query over the owned repository client. Seeded at
+ // InitializeAsync and re-seeded on every deploy-watcher rediscovery: the runtime never calls
+ // DiscoverAsync on a live driver (only the /raw picker's throwaway does), so SyncPlatformsAsync
+ // from DiscoverAsync alone left the production watcher with no platforms, ever.
+ private readonly IGalaxyPlatformSource? _platformSource;
+ private IGalaxyPlatformSource? _ownedPlatformSource;
+ // Probe-seed requests (init + every rediscovery) not yet served. The first request starts the
+ // background worker; later ones only bump the count, and the worker loops until it has served
+ // every request it saw — so a request that lands while a pass is mid-fetch (a deploy, or the
+ // next runtime's init after a teardown) still gets a pass that starts after it.
+ private int _reseedRequests;
+ private volatile string _seedReason = "initialize";
+
+ // Cancels the in-flight background seed of the CURRENT runtime. Created by InitializeAsync,
+ // cancelled (never disposed — it has no timer or wait handle, and the worker may still read its
+ // token) by the teardown.
+ private CancellationTokenSource? _seedCts;
+
+ // Serialises "is this registration still for the live runtime?" against the teardown's registry
+ // Clear, so a subscribe / re-subscribe whose SubscribeBulk straddled a teardown either lands
+ // before the Clear (and is wiped by it) or sees the bumped generation and backs out.
+ private readonly Lock _registryGate = new();
+
+ // Bumped by every runtime teardown. Background work started under one runtime (the replay-gap
+ // re-snapshot, a rediscovery re-seed) re-checks it before touching the registry / watcher, so a
+ // late Rebind cannot resurrect handles into a registry the teardown already cleared.
+ private int _runtimeGeneration;
+
+ ///
+ /// Upper bound on the gateway CloseSession round-trip during a teardown. The SDK's
+ /// default close runs under the 30 s call timeout, which against a blackholed gateway parks
+ /// ShutdownAsync — and the host's PostStop — for the whole of it. Internal so
+ /// tests can shrink it.
+ ///
+ internal TimeSpan SessionCloseDeadline { get; set; } = TimeSpan.FromSeconds(5);
+
+ // True while the seams (_subscriber / _dataWriter / _hierarchySource / _alarmFeed /
+ // _alarmAcknowledger) were built by BuildProductionRuntimeAsync and are therefore ours to dispose
+ // and null on ShutdownAsync — the host's comms-loss recovery is ShutdownAsync → InitializeAsync on
+ // this SAME object, and InitializeAsync rebuilds only when the seams are null. Injected seams
+ // belong to the injector: ShutdownAsync stops every loop it started on top of them but leaves the
+ // seams in place, so a seam-driven test can cycle the lifecycle the way the watchdog does.
+ private bool _ownsRuntime;
+
+ // Serialises InitializeAsync / ShutdownAsync / DisposeAsync so a late watchdog re-init can never
+ // interleave with a teardown and resurrect half of the runtime it is tearing down.
+ private readonly SemaphoreSlim _lifecycleGate = new(1, 1);
+
+ // Set for the duration of a runtime teardown so the pump's stream-fault / replay-gap callbacks and
+ // the supervisor's replay cannot start a fresh pump underneath it.
+ private volatile bool _runtimeStopping;
+
private DriverHealth _health = new(DriverState.Unknown, null, null);
private bool _disposed;
+ ///
+ /// Test seam — forwarded to on the
+ /// session creates, so the owned-runtime lifecycle
+ /// (client + session construction, teardown, rebuild) can be exercised without a gateway. The
+ /// gRPC channels it builds are real but lazy: nothing dials until a call is made.
+ ///
+ internal Func? SessionOpenOverrideForTests { get; set; }
+
+ /// Test seam: the owned gateway client, null when no production runtime is live.
+ internal MxGatewayClient? OwnedMxClientForTests => _ownedMxClient;
+
+ /// Test seam: the owned MX session, null when no production runtime is live.
+ internal GalaxyMxSession? OwnedMxSessionForTests => _ownedMxSession;
+
+ /// Test seam: the owned repository client, null until the deploy watcher or discovery built it.
+ internal GalaxyRepositoryClient? OwnedRepositoryClientForTests => _ownedRepositoryClient;
+
+ /// Test seam: the live reconnect supervisor, null outside a production runtime.
+ internal ReconnectSupervisor? SupervisorForTests => _supervisor;
+
+ /// Test seam: the live per-platform probe watcher, null until InitializeAsync.
+ internal PerPlatformProbeWatcher? ProbeWatcherForTests => _probeWatcher;
+
+ /// Test seam: the live deploy watcher, null until InitializeAsync starts one.
+ internal DeployWatcher? DeployWatcherForTests => _deployWatcher;
+
+ /// Test seam: true while an is running.
+ internal bool HasEventPumpForTests { get { lock (_pumpLock) return _eventPump is not null; } }
+
+ /// Test seam: the number of driver subscriptions the registry still tracks.
+ internal int TrackedSubscriptionCountForTests => _subscriptions.TrackedSubscriptionCount;
+
///
/// Server-pushed data-change notification. Fires from the
/// 's background loop; handlers should be cheap (or queue
@@ -202,6 +291,15 @@ public sealed class GalaxyDriver
/// null-object resolver (returns null for every name) so seam-injecting tests that
/// don't exercise a secret: ref need not supply one.
///
+ ///
+ /// Optional deploy-event stream seam. When supplied, starts the
+ /// over it (production builds one over the owned repository client).
+ ///
+ ///
+ /// Optional platform-set seam for the ScanState probe watcher. When supplied,
+ /// seeds the watcher from it and every rediscovery re-seeds
+ /// (production builds a over the owned repository client).
+ ///
internal GalaxyDriver(
string driverInstanceId,
GalaxyDriverOptions options,
@@ -212,8 +310,12 @@ public sealed class GalaxyDriver
IGalaxyAlarmAcknowledger? alarmAcknowledger = null,
IGalaxyAlarmFeed? alarmFeed = null,
ILogger? logger = null,
- ISecretResolver? secretResolver = null)
+ ISecretResolver? secretResolver = null,
+ IGalaxyDeployWatchSource? deployWatchSource = null,
+ IGalaxyPlatformSource? platformSource = null)
{
+ _deployWatchSource = deployWatchSource;
+ _platformSource = platformSource;
_driverInstanceId = !string.IsNullOrWhiteSpace(driverInstanceId)
? driverInstanceId
: throw new ArgumentException("Driver instance id required.", nameof(driverInstanceId));
@@ -359,28 +461,195 @@ public sealed class GalaxyDriver
ObjectDisposedException.ThrowIf(_disposed, this);
if (HasConfigBody(driverConfigJson)) RefuseNonEquivalentConfig(driverConfigJson);
- // Tests inject seams via the internal ctor; production InitializeAsync builds
- // the gateway client + session + per-capability runtime components from
- // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the
- // test exercises the wired surface without a real gw round-trip).
- if (_subscriber is null && _dataWriter is null && _hierarchySource is null)
+ await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
{
- await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false);
+ ObjectDisposedException.ThrowIf(_disposed, this);
+
+ // Tests inject seams via the internal ctor; production InitializeAsync builds
+ // the gateway client + session + per-capability runtime components from
+ // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the
+ // test exercises the wired surface without a real gw round-trip). After a
+ // ShutdownAsync the production seams are null again, so the watchdog's
+ // Shutdown → Initialize genuinely rebuilds the runtime on this same object.
+ if (_subscriber is null && _dataWriter is null && _hierarchySource is null)
+ {
+ await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false);
+ }
+ else
+ {
+ _logger.LogDebug(
+ "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped",
+ _driverInstanceId);
+ // The per-platform probe watcher needs only a subscriber, so it runs on the seam path
+ // too — that is what lets the ScanState wiring be exercised without a gateway.
+ if (_subscriber is not null) _probeWatcher ??= BuildProbeWatcher(_subscriber);
+ }
+
+ await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false);
+ // The platform seed is a gateway DiscoverHierarchy round-trip, which can spend the whole of
+ // a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It
+ // runs in the background: InitializeAsync neither waits on it nor fails because of it.
+ await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false);
+ RequestProbeSeed("initialize");
+ _logger.LogInformation(
+ "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
+ _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName);
+ _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
}
- else
+ finally
{
- _logger.LogDebug(
- "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped",
+ _lifecycleGate.Release();
+ }
+ }
+
+ ///
+ /// Under the lifecycle gate, before the first seed request of a runtime: resolve the platform
+ /// source (production builds it over the owned repository client — no network here) and arm
+ /// the runtime's seed cancellation. Resolving here, not in the worker, keeps the background
+ /// pass from creating a repository client after a teardown released it. A failure is a
+ /// Warning: the value path does not depend on the probe set.
+ ///
+ private async Task PrepareProbeSeedAsync(CancellationToken cancellationToken)
+ {
+ if (_probeWatcher is null) return;
+ _seedCts ??= new CancellationTokenSource();
+ if (_platformSource is not null || !_ownsRuntime || _ownedPlatformSource is not null) return;
+ try
+ {
+ _ownedRepositoryClient ??= GalaxyRepositoryClient.Create(
+ await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false));
+ _ownedPlatformSource =
+ new GatewayGalaxyPlatformSource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
+ {
+ _logger.LogWarning(ex,
+ "GalaxyDriver {InstanceId} could not build the platform query — per-platform host status stays unseeded until the next rediscovery.",
_driverInstanceId);
}
-
- await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false);
- _logger.LogInformation(
- "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
- _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName);
- _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
}
+ ///
+ /// Ask for a background probe seed (init and every deploy-watcher rediscovery). Requests
+ /// coalesce onto one worker; each worker pass serves the runtime that is live when the pass
+ /// STARTS, and a pass whose fetch completes after a teardown is discarded — so a request made
+ /// under a new runtime always gets a pass of its own.
+ ///
+ private void RequestProbeSeed(string reason)
+ {
+ if (_runtimeStopping || _probeWatcher is null) return;
+ _seedReason = reason;
+ if (Interlocked.Increment(ref _reseedRequests) != 1) return; // the running worker will serve it
+
+ _ = Task.Run(async () =>
+ {
+ int served;
+ do
+ {
+ served = Volatile.Read(ref _reseedRequests);
+ var generation = Volatile.Read(ref _runtimeGeneration);
+ var seedCts = Volatile.Read(ref _seedCts);
+ if (!_runtimeStopping && seedCts is not null)
+ {
+ await SeedProbePlatformsAsync(_seedReason, generation, seedCts.Token).ConfigureAwait(false);
+ }
+ }
+ while (Interlocked.Add(ref _reseedRequests, -served) != 0);
+ });
+ }
+
+ ///
+ /// One seed pass: fetch the platform / engine set and hand it to the probe watcher of the
+ /// runtime identified by . Never throws — a failed or cancelled
+ /// fetch logs and leaves the watcher as it was (the next rediscovery retries), and a result
+ /// that lands after that runtime was torn down is discarded.
+ ///
+ private async Task SeedProbePlatformsAsync(string reason, int generation, CancellationToken cancellationToken)
+ {
+ var watcher = _probeWatcher;
+ var source = _platformSource ?? _ownedPlatformSource;
+ if (watcher is null || source is null) return;
+
+ try
+ {
+ var platforms = await source.GetPlatformTagNamesAsync(cancellationToken).ConfigureAwait(false);
+ if (Volatile.Read(ref _runtimeGeneration) != generation || _runtimeStopping
+ || !ReferenceEquals(watcher, _probeWatcher))
+ {
+ _logger.LogDebug(
+ "GalaxyDriver {InstanceId} discarded a platform seed ({Reason}) — its runtime was torn down.",
+ _driverInstanceId, reason);
+ return;
+ }
+ await watcher.SyncPlatformsAsync(platforms, cancellationToken).ConfigureAwait(false);
+ _logger.LogInformation(
+ "GalaxyDriver {InstanceId} probe watcher seeded with {Count} platform(s) ({Reason}).",
+ _driverInstanceId, platforms.Count, reason);
+ }
+ catch (Exception ex) when (cancellationToken.IsCancellationRequested
+ || Volatile.Read(ref _runtimeGeneration) != generation)
+ {
+ _logger.LogDebug(ex, "GalaxyDriver {InstanceId} platform seed ({Reason}) abandoned by a teardown.",
+ _driverInstanceId, reason);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "GalaxyDriver {InstanceId} platform seed failed ({Reason}) — per-platform host status stays as it was until the next rediscovery.",
+ _driverInstanceId, reason);
+ }
+ }
+
+ /// Deploy-watcher rediscovery: a redeploy may add or remove platforms / engines.
+ private void ReseedProbePlatformsOnRediscovery() => RequestProbeSeed("rediscovery");
+
+ ///
+ /// Register only while is still the
+ /// live runtime. Atomic with the teardown's registry Clear (both under ).
+ ///
+ private bool TryRegisterForGeneration(long subscriptionId, IReadOnlyList bindings, int generation)
+ {
+ lock (_registryGate)
+ {
+ if (Volatile.Read(ref _runtimeGeneration) != generation) return false;
+ _subscriptions.Register(subscriptionId, bindings);
+ return true;
+ }
+ }
+
+ private void ClearRegistry()
+ {
+ lock (_registryGate) _subscriptions.Clear();
+ }
+
+ ///
+ /// Best-effort release of the live handles a SubscribeBulk produced for a call that then lost
+ /// its runtime — the subscriber may already be dead, so a failure is logged and swallowed.
+ ///
+ private async Task ReleaseOrphanedHandlesAsync(IGalaxySubscriber subscriber, IReadOnlyList bindings)
+ {
+ var live = bindings.Where(b => b.ItemHandle > 0).Select(b => b.ItemHandle).ToArray();
+ if (live.Length == 0) return;
+ try { await subscriber.UnsubscribeBulkAsync(live, CancellationToken.None).ConfigureAwait(false); }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "GalaxyDriver {InstanceId} could not release {Count} handle(s) orphaned by a teardown.",
+ _driverInstanceId, live.Length);
+ }
+ }
+
+ private static InvalidOperationException TornDownDuringSubscribe() => new(
+ "GalaxyDriver runtime was torn down while the subscribe was in flight — retry after the next InitializeAsync.");
+
+ // The watcher publishes its probe handles into the driver's registry (reserved id) so the
+ // EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every
+ // probe event was dropped before it got anywhere near the watcher.
+ private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) =>
+ new(subscriber, _hostStatuses, _subscriptions, _logger,
+ bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
+
///
/// Build the production gw client + session + per-capability runtime components
/// from _options. Sets up the reconnect supervisor's reopen / replay
@@ -389,9 +658,45 @@ public sealed class GalaxyDriver
///
private async Task BuildProductionRuntimeAsync(CancellationToken cancellationToken)
{
+ try
+ {
+ await BuildProductionRuntimeCoreAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex)
+ {
+ // A failed connect must not leave a half-built runtime behind: the watchdog retries
+ // InitializeAsync every 10 s, and the seam check above would see the orphaned client +
+ // session as "already built" — or, worse, rebuild on top of them. Release what was
+ // created, and land the failed state before AND after the teardown (IDriver remark).
+ _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message);
+ await TeardownRuntimeAsync(final: false, CancellationToken.None).ConfigureAwait(false);
+ _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message);
+ throw;
+ }
+ }
+
+ private async Task BuildProductionRuntimeCoreAsync(CancellationToken cancellationToken)
+ {
+ // A pump that slipped in between a teardown and this rebuild (a late SubscribeAsync) is bound
+ // to the dead subscriber; it must never be reused as this runtime's pump.
+ EventPump? stale;
+ lock (_pumpLock) { stale = _eventPump; _eventPump = null; }
+ if (stale is not null)
+ {
+ stale.OnDataChange -= OnPumpDataChange;
+ try { await stale.DisposeAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "Stale EventPump dispose failed before runtime rebuild"); }
+ }
+ // Unconditionally: whatever the registry holds names handles of a session this rebuild replaces.
+ ClearRegistry();
+
+ _ownsRuntime = true;
var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false);
_ownedMxClient = MxGatewayClient.Create(clientOptions);
- _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger);
+ _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger)
+ {
+ OpenAndRegisterOverrideForTests = SessionOpenOverrideForTests,
+ };
await _ownedMxSession.ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false);
// PR 6.1 — wrap the gw-facing seams in tracing decorators so every Subscribe /
@@ -424,9 +729,7 @@ public sealed class GalaxyDriver
_supervisor.StateChanged += OnSupervisorStateChanged;
- _probeWatcher = new PerPlatformProbeWatcher(
- _subscriber, _hostStatuses, _logger,
- bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
+ _probeWatcher = BuildProbeWatcher(_subscriber);
// Wire the alarm acknowledger + feed to the live gateway client. Both are
// session-less — the gateway serves alarms from an always-on central monitor —
@@ -436,6 +739,143 @@ public sealed class GalaxyDriver
_ownedMxClient.StreamAlarmsAsync, _logger, _options.MxAccess.ClientName);
}
+ ///
+ /// Stop every background loop and release every gateway resource this driver holds — the
+ /// teardown half shared by ( = false:
+ /// a later rebuilds) and
+ /// ( = true). Before this existed, ShutdownAsync was a
+ /// no-op and everything below outlived the driver on every respawn, removal and
+ /// /raw browse: the gateway accumulated live MX sessions and advised items, and
+ /// this process accumulated channels and stream loops.
+ ///
+ ///
+ /// Ordering matters:
+ ///
+ /// The supervisor goes first and is awaited, so no recovery loop can reopen the
+ /// session or start a pump while the rest is being released.
+ /// The probe watcher unsubscribes while the session is still open (its handles
+ /// die with the session anyway, but a clean release keeps the gateway's advise
+ /// count honest).
+ /// Streams (pump, alarm feed, deploy watcher) are cancelled and awaited.
+ /// The session closes (gateway CloseSession), then both owned channels.
+ ///
+ /// Every step is best-effort: one failing release never prevents the next.
+ ///
+ private async Task TeardownRuntimeAsync(bool final, CancellationToken cancellationToken)
+ {
+ _runtimeStopping = true;
+ Interlocked.Increment(ref _runtimeGeneration);
+ Interlocked.Exchange(ref _seedCts, null)?.Cancel(); // a seed in flight belongs to this runtime
+ try
+ {
+ var supervisor = _supervisor;
+ _supervisor = null;
+ if (supervisor is not null)
+ {
+ supervisor.StateChanged -= OnSupervisorStateChanged;
+ try { await supervisor.StopAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor stop failed"); }
+ }
+
+ var deployWatcher = _deployWatcher;
+ _deployWatcher = null;
+ if (deployWatcher is not null)
+ {
+ deployWatcher.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded;
+ try { await deployWatcher.StopAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher stop failed"); }
+ }
+
+ var probeWatcher = _probeWatcher;
+ _probeWatcher = null;
+ if (probeWatcher is not null)
+ {
+ try { await probeWatcher.StopAsync(cancellationToken).ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher stop failed"); }
+ }
+
+ try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); }
+ _transportForwarder = null;
+
+ EventPump? pump;
+ lock (_pumpLock) { pump = _eventPump; _eventPump = null; }
+ if (pump is not null)
+ {
+ pump.OnDataChange -= OnPumpDataChange;
+ try { await pump.DisposeAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); }
+ }
+ // Every item handle the registry knows died with the pump's session; the host
+ // re-subscribes (fresh ids) once the next InitializeAsync reports Connected.
+ ClearRegistry();
+
+ // The alarm feed: owned → disposed and nulled so InitializeAsync builds a fresh one;
+ // injected → detached (the injector owns its lifetime) unless this is the final dispose.
+ IGalaxyAlarmFeed? alarmFeed;
+ bool releaseFeed;
+ lock (_alarmFeedLock)
+ {
+ alarmFeed = _alarmFeed;
+ releaseFeed = alarmFeed is not null && (_ownsRuntime || final);
+ if (alarmFeed is not null && _alarmFeedWired)
+ {
+ alarmFeed.OnAlarmTransition -= OnAlarmFeedTransition;
+ _alarmFeedWired = false;
+ }
+ if (releaseFeed) _alarmFeed = null;
+ }
+ if (releaseFeed)
+ {
+ try { await alarmFeed!.DisposeAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); }
+ }
+
+ if (_ownedMxSession is not null)
+ {
+ var session = _ownedMxSession;
+ _ownedMxSession = null;
+ // Bounded close: the caller's token plus SessionCloseDeadline, so a blackholed gateway
+ // cannot park this (and the host's PostStop) for the SDK's 30 s call timeout.
+ using var closeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ closeCts.CancelAfter(SessionCloseDeadline);
+ try { await session.CloseAsync(closeCts.Token).ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "MxSession close failed"); }
+ try { await session.DisposeAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); }
+ }
+
+ if (_ownedMxClient is not null)
+ {
+ try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); }
+ _ownedMxClient = null;
+ }
+
+ if (_ownedRepositoryClient is not null)
+ {
+ try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); }
+ _ownedRepositoryClient = null;
+ }
+
+ if (_ownsRuntime || final)
+ {
+ // Null the seams so the next InitializeAsync rebuilds (production) — injected seams
+ // are only dropped on the final dispose.
+ _subscriber = null;
+ _dataWriter = null;
+ _hierarchySource = null;
+ _alarmAcknowledger = null;
+ _ownedPlatformSource = null;
+ _ownsRuntime = false;
+ }
+ }
+ finally
+ {
+ _runtimeStopping = false;
+ }
+ }
+
///
/// Reopen callback for : recreate the gw session. After a
/// gateway restart the existing session handle is faulted / stale but non-null, so a plain
@@ -465,6 +905,9 @@ public sealed class GalaxyDriver
///
private async Task ReplayAsync(CancellationToken cancellationToken)
{
+ // A teardown cancels the supervisor's cycle between reopen and replay; a replay that ran on
+ // anyway would start a fresh pump underneath the shutdown.
+ cancellationToken.ThrowIfCancellationRequested();
if (_subscriber is null) return;
var entries = _subscriptions.SnapshotEntries();
if (entries.Count == 0) return;
@@ -488,7 +931,8 @@ public sealed class GalaxyDriver
// completed loop. Recreate the pump so the replayed subscriptions have a consumer.
RestartEventPumpForReplay();
- var tagCount = await ResubscribeAllAsync(entries, cancellationToken).ConfigureAwait(false);
+ var tagCount = await ResubscribeAllAsync(entries, Volatile.Read(ref _runtimeGeneration), cancellationToken)
+ .ConfigureAwait(false);
_logger.LogInformation(
"GalaxyDriver {InstanceId} replay completed — {SubCount} subscriptions, {TagCount} tags re-subscribed",
@@ -502,14 +946,37 @@ public sealed class GalaxyDriver
/// replay must recreate (its loop died with the stream) and a gap must NOT touch (the
/// stream carrying the sentinel is still healthy and still delivering).
///
+ /// The subscriptions to re-issue.
+ ///
+ /// The runtime generation this re-subscribe belongs to. Re-checked before every
+ /// : a teardown bumps it and clears the registry,
+ /// and a Rebind landing after that would resurrect dead handles into the next runtime.
+ ///
+ /// Cancellation token.
/// The number of tag references re-subscribed.
private async Task ResubscribeAllAsync(
IReadOnlyList<(long SubscriptionId, IReadOnlyList Bindings)> entries,
+ int generation,
CancellationToken cancellationToken)
{
var tagCount = 0;
foreach (var (subscriptionId, oldBindings) in entries)
{
+ if (Volatile.Read(ref _runtimeGeneration) != generation)
+ {
+ _logger.LogDebug("GalaxyDriver {InstanceId} re-subscribe abandoned — runtime was torn down.", _driverInstanceId);
+ return tagCount;
+ }
+
+ if (subscriptionId == SubscriptionRegistry.ProbeSubscriptionId && _probeWatcher is { } probeWatcher)
+ {
+ // The probe watcher owns its handles: it re-subscribes its platforms, records the
+ // fresh handles and rebinds the reserved registry entry itself, so ScanState host
+ // status keeps updating after a reconnect (it used to go permanently deaf).
+ tagCount += await probeWatcher.ResubscribeAsync(cancellationToken).ConfigureAwait(false);
+ continue;
+ }
+
var refs = oldBindings
.Select(b => b.FullReference)
.Distinct(StringComparer.OrdinalIgnoreCase)
@@ -531,8 +998,13 @@ public sealed class GalaxyDriver
}
// Rebind so the EventPump fan-out reverse map points at the post-reconnect
- // handles; otherwise events on the new handles miss every subscription.
- _subscriptions.Rebind(subscriptionId, newBindings);
+ // handles; otherwise events on the new handles miss every subscription. Not after a
+ // teardown: the fresh handles belong to a subscriber that is already being released.
+ lock (_registryGate)
+ {
+ if (Volatile.Read(ref _runtimeGeneration) != generation) return tagCount;
+ _subscriptions.Rebind(subscriptionId, newBindings);
+ }
tagCount += refs.Length;
}
@@ -581,9 +1053,9 @@ public sealed class GalaxyDriver
}
// EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the
- // recreate if the driver is being disposed — Dispose already tore the pump down
- // and a fresh one here would leak past the driver's lifetime.
- if (_disposed) return;
+ // recreate if the driver is being disposed or its runtime torn down — the teardown
+ // already released the pump and a fresh one here would leak past it.
+ if (_disposed || _runtimeStopping) return;
EnsureEventPumpStarted();
}
@@ -631,28 +1103,57 @@ public sealed class GalaxyDriver
private async Task StartDeployWatcherAsync(CancellationToken cancellationToken)
{
if (!_options.Repository.WatchDeployEvents) return;
- if (_ownedRepositoryClient is null && _hierarchySource is null) return;
- // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand).
- // If discovery hasn't run yet, build the client here so the watcher has a target.
- // Build under a null-check (not ??=) so if BuildDefaultHierarchySourceAsync later
- // runs it reuses this client rather than overwriting the field and leaking the
- // first instance — the client-options build is now async (secret: arm).
- _ownedRepositoryClient ??= ZB.MOM.WW.MxGateway.Client.GalaxyRepositoryClient.Create(
- await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false));
+ var source = _deployWatchSource;
+ if (source is null)
+ {
+ // Seam mode without an injected watch source has no gateway to watch. (Production
+ // always owns its runtime here.) The old guard returned whenever BOTH the repository
+ // client and the hierarchy source were null — which on a production runtime instance
+ // is every first InitializeAsync, since discovery runs on the /raw picker's throwaway
+ // instance, not here — so the watcher never started and the re-browse chip never
+ // fired for a live driver.
+ if (!_ownsRuntime) return;
- var source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient);
- _deployWatcher = new DeployWatcher(source, _logger);
- _deployWatcher.OnRediscoveryNeeded += (_, args) => OnRediscoveryNeeded?.Invoke(this, args);
+ // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand).
+ // Build under a null-check (not an overwrite) so if BuildDefaultHierarchySourceAsync
+ // later runs it reuses this client rather than leaking the first instance.
+ _ownedRepositoryClient ??= GalaxyRepositoryClient.Create(
+ await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false));
+ source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient);
+ }
+
+ // One watcher per driver: a previous one (an InitializeAsync without a ShutdownAsync in
+ // between) is stopped before its replacement starts — every live InitializeAsync used to
+ // stack another watch-stream loop, each raising OnRediscoveryNeeded per real deploy.
+ var previous = _deployWatcher;
+ _deployWatcher = null;
+ if (previous is not null)
+ {
+ previous.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded;
+ try { await previous.StopAsync().ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "Previous DeployWatcher stop failed — replacing it anyway."); }
+ }
+
+ var watcher = new DeployWatcher(source, _logger);
+ watcher.OnRediscoveryNeeded += OnDeployWatcherRediscoveryNeeded;
+ _deployWatcher = watcher;
// StartAsync schedules the background loop and returns Task.CompletedTask immediately.
// It throws InvalidOperationException synchronously if called twice (programming error).
// Don't discard the return value — observe any synchronous throw.
- var startTask = _deployWatcher.StartAsync(CancellationToken.None);
+ var startTask = watcher.StartAsync(CancellationToken.None);
// The task is already completed (StartAsync is synchronous); surface any synchronous fault.
if (startTask.IsFaulted) startTask.GetAwaiter().GetResult();
}
+ private void OnDeployWatcherRediscoveryNeeded(object? sender, RediscoveryEventArgs args)
+ {
+ OnRediscoveryNeeded?.Invoke(this, args);
+ // A redeploy may add or remove platforms / engines; refresh the ScanState probe set.
+ ReseedProbePlatformsOnRediscovery();
+ }
+
///
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
{
@@ -731,13 +1232,32 @@ public sealed class GalaxyDriver
&& a.Reconnect == b.Reconnect
&& a.ProbeTimeoutSeconds == b.ProbeTimeoutSeconds;
+ ///
+ /// Stop the driver and release the gateway runtime: supervisor, deploy watcher, probe
+ /// watcher, event pump, alarm feed, MX session (gateway CloseSession) and both owned
+ /// gRPC channels. The seams are nulled afterwards, so a later
+ /// on this same object — the host's comms-loss recovery and the way ReinitializeAsync
+ /// is sequenced — rebuilds a fresh runtime instead of reusing a dead one. The driver object
+ /// itself stays usable; is the final, idempotent teardown.
+ ///
///
- public Task ShutdownAsync(CancellationToken cancellationToken)
+ public async Task ShutdownAsync(CancellationToken cancellationToken)
{
- if (_disposed) return Task.CompletedTask;
+ if (_disposed) return;
_logger.LogInformation("GalaxyDriver {InstanceId} shutting down", _driverInstanceId);
- _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
- return Task.CompletedTask;
+ await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ if (_disposed) return;
+ await TeardownRuntimeAsync(final: false, cancellationToken).ConfigureAwait(false);
+ // Terminal health lands AFTER the teardown (IDriver remark): a loop unwinding during the
+ // release could otherwise overwrite it with the state it was reporting when cancelled.
+ _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
+ }
+ finally
+ {
+ _lifecycleGate.Release();
+ }
}
///
@@ -786,11 +1306,12 @@ public sealed class GalaxyDriver
// keyed by RawPath, matching what WriteAsync resolves against.
var discoverer = new GalaxyDiscoverer(
source, attributeRef => _rawTagByAttributeRef.GetValueOrDefault(attributeRef));
- await discoverer.DiscoverAsync(capturingBuilder, cancellationToken).ConfigureAwait(false);
+ // One sweep: the discoverer hands back the hierarchy it materialised, and the platform
+ // selection below reads that instead of asking the gateway for the whole galaxy again.
+ var hierarchy = await discoverer.DiscoverAsync(capturingBuilder, cancellationToken).ConfigureAwait(false);
if (_probeWatcher is not null)
{
- var hierarchy = await source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false);
var platforms = hierarchy
.Where(o => o.TemplateChain.Any(t =>
string.Equals(t, "$WinPlatform", StringComparison.OrdinalIgnoreCase)
@@ -862,6 +1383,8 @@ public sealed class GalaxyDriver
deadlineCts.CancelAfter(ReadViaSubscribeDeadline);
var cancellationToken = deadlineCts.Token;
+ var generation = Volatile.Read(ref _runtimeGeneration); // see SubscribeAsync
+ var readSubscriber = _subscriber!;
var pump = EnsureEventPumpStarted();
var subscriptionId = _subscriptions.NextSubscriptionId();
@@ -895,7 +1418,7 @@ public sealed class GalaxyDriver
// The RPC gets only the CALLER's token: the internal deadline bounds our wait, not the RPC, so a slow
// gateway can never surface an OperationCanceledException to a caller that did not cancel, and the
// handles it eventually creates are known (and unsubscribed below) rather than lost to a cancelled call.
- var subscribeTask = _subscriber!.SubscribeBulkAsync(fullReferences, bufferedIntervalMs, callerToken);
+ var subscribeTask = readSubscriber.SubscribeBulkAsync(fullReferences, bufferedIntervalMs, callerToken);
try
{
results = await subscribeTask.WaitAsync(cancellationToken).ConfigureAwait(false);
@@ -904,7 +1427,7 @@ public sealed class GalaxyDriver
{
// Internal deadline fired while the subscribe RPC was still in flight.
pump.OnDataChange -= handler;
- var subscriber = _subscriber!;
+ var subscriber = readSubscriber;
var logger = _logger;
_ = subscribeTask.ContinueWith(t =>
{
@@ -956,7 +1479,12 @@ public sealed class GalaxyDriver
ServerTimestampUtc: DateTime.UtcNow));
}
}
- _subscriptions.Register(subscriptionId, bindings);
+ if (!TryRegisterForGeneration(subscriptionId, bindings, generation))
+ {
+ pump.OnDataChange -= handler;
+ await ReleaseOrphanedHandlesAsync(readSubscriber, bindings).ConfigureAwait(false);
+ throw TornDownDuringSubscribe();
+ }
try
{
@@ -995,7 +1523,7 @@ public sealed class GalaxyDriver
{
try
{
- await _subscriber!.UnsubscribeBulkAsync(liveHandles, CancellationToken.None)
+ await readSubscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -1056,6 +1584,10 @@ public sealed class GalaxyDriver
"Call InitializeAsync and wait for a successful connection before subscribing.");
}
+ // Captured before the pump/stopping check: a teardown that starts after this point bumps the
+ // generation, so the Register below can tell its SubscribeBulk straddled a teardown.
+ var generation = Volatile.Read(ref _runtimeGeneration);
+ var subscriber = _subscriber;
var pump = EnsureEventPumpStarted();
var subscriptionId = _subscriptions.NextSubscriptionId();
@@ -1063,7 +1595,7 @@ public sealed class GalaxyDriver
{
// Empty subscriptions register but never bind anything — keeps Unsubscribe
// symmetric for callers that conditionally add tags later.
- _subscriptions.Register(subscriptionId, []);
+ if (!TryRegisterForGeneration(subscriptionId, [], generation)) throw TornDownDuringSubscribe();
return new GalaxySubscriptionHandle(subscriptionId);
}
@@ -1079,7 +1611,7 @@ public sealed class GalaxyDriver
// and infrastructure callers (probe watcher, deploy watcher) hit the fallback.
var requested = (int)Math.Max(0, publishingInterval.TotalMilliseconds);
var bufferedIntervalMs = requested > 0 ? requested : _options.MxAccess.PublishingIntervalMs;
- var results = await _subscriber
+ var results = await subscriber
.SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken)
.ConfigureAwait(false);
@@ -1107,7 +1639,13 @@ public sealed class GalaxyDriver
}
}
- _subscriptions.Register(subscriptionId, bindings);
+ if (!TryRegisterForGeneration(subscriptionId, bindings, generation))
+ {
+ // The teardown cleared the registry while SubscribeBulk was in flight; registering now
+ // would carry dead handles into the next runtime. Release what we got and fail the call.
+ await ReleaseOrphanedHandlesAsync(subscriber, bindings).ConfigureAwait(false);
+ throw TornDownDuringSubscribe();
+ }
_ = pump; // keep the pump alive for the subscription's lifetime
return new GalaxySubscriptionHandle(subscriptionId);
}
@@ -1152,6 +1690,15 @@ public sealed class GalaxyDriver
{
lock (_pumpLock)
{
+ // Checked BEFORE the reuse: the flag is set before the teardown takes the pump under this
+ // same lock, so while it is set the live pump belongs to a dying subscriber. Handing it out
+ // would register handles into a registry the teardown is about to clear; creating a new one
+ // would leave a pump on the dead subscriber for the next InitializeAsync to inherit.
+ if (_runtimeStopping)
+ {
+ throw new InvalidOperationException(
+ "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync.");
+ }
if (_eventPump is not null) return _eventPump;
_eventPump = new EventPump(
_subscriber!, _subscriptions, _logger,
@@ -1192,7 +1739,7 @@ public sealed class GalaxyDriver
///
private void OnEventPumpReplayGap(ReplayGap gap)
{
- if (_subscriber is null) return;
+ if (_subscriber is null || _runtimeStopping) return;
if (!_options.Reconnect.ReplayOnSessionLost)
{
@@ -1212,6 +1759,7 @@ public sealed class GalaxyDriver
return;
}
+ var generation = Volatile.Read(ref _runtimeGeneration);
_ = Task.Run(async () =>
{
try
@@ -1219,7 +1767,7 @@ public sealed class GalaxyDriver
var entries = _subscriptions.SnapshotEntries();
if (entries.Count == 0) return;
- var tagCount = await ResubscribeAllAsync(entries, CancellationToken.None).ConfigureAwait(false);
+ var tagCount = await ResubscribeAllAsync(entries, generation, CancellationToken.None).ConfigureAwait(false);
_logger.LogInformation(
"GalaxyDriver {InstanceId} replay-gap re-snapshot completed — {SubCount} subscriptions, {TagCount} tags re-subscribed",
_driverInstanceId, entries.Count, tagCount);
@@ -1249,6 +1797,14 @@ public sealed class GalaxyDriver
///
private void OnEventPumpStreamFault(Exception cause)
{
+ if (_runtimeStopping)
+ {
+ // The pump is being torn down with the rest of the runtime — a fault it reports on the
+ // way out is shutdown noise, not something to recover from.
+ _logger.LogDebug(cause, "GalaxyDriver {InstanceId} event stream ended during runtime teardown.", _driverInstanceId);
+ return;
+ }
+
var supervisor = _supervisor;
if (supervisor is null)
{
@@ -1464,6 +2020,24 @@ public sealed class GalaxyDriver
///
private void OnPumpDataChange(object? sender, DataChangeEventArgs args)
{
+ // Anything the gateway pushes on the stream proves it is alive and serving this session —
+ // the comms-loss watchdog's evidence (G-11: the stamp used to move only at init and on a
+ // supervisor Healthy transition, so a 90 s Degraded tripped a re-init under flowing data).
+ StampLiveness();
+
+ // Probe traffic rides the reserved registry entry and stays in the attributeRef domain —
+ // ScanState attributes are discovered platform attributes, never authored RawPaths. It is
+ // the watcher's alone: the host never subscribed it and must not see it on OnDataChange.
+ if (args.SubscriptionHandle is GalaxySubscriptionHandle { SubscriptionId: SubscriptionRegistry.ProbeSubscriptionId })
+ {
+ // The probe decoder takes a raw quality byte. Recover it via the canonical
+ // StatusCodeMap.ToQualityCategoryByte helper so the mapping lives in one
+ // place next to its inverse (FromQualityByte) and cannot desync silently.
+ var qualityByte = StatusCodeMap.ToQualityCategoryByte(args.Snapshot.StatusCode);
+ _probeWatcher?.OnProbeValueChanged(args.FullReference, args.Snapshot.Value, qualityByte);
+ return;
+ }
+
// v3: the pump fans out keyed by the dialled attributeRef; the server correlates values to nodes
// by RawPath (the node's FullName). Reverse-map before surfacing to public OnDataChange consumers.
// Identity on a miss, so a live-browse tag surfaces under its attributeRef unchanged.
@@ -1474,18 +2048,25 @@ public sealed class GalaxyDriver
publicArgs = args with { FullReference = rawPath };
}
OnDataChange?.Invoke(this, publicArgs);
+ }
- // Probe watching stays in the attributeRef domain — ScanState attributes are discovered platform
- // attributes, never authored RawPaths, so use the untranslated reference here.
- if (_probeWatcher is not null
- && args.FullReference.EndsWith(PerPlatformProbeWatcher.ProbeSuffix, StringComparison.OrdinalIgnoreCase))
- {
- // The probe decoder takes a raw quality byte. Recover it via the canonical
- // StatusCodeMap.ToQualityCategoryByte helper so the mapping lives in one
- // place next to its inverse (FromQualityByte) and cannot desync silently.
- var qualityByte = StatusCodeMap.ToQualityCategoryByte(args.Snapshot.StatusCode);
- _probeWatcher.OnProbeValueChanged(args.FullReference, args.Snapshot.Value, qualityByte);
- }
+ /// Minimum gap between two LastSuccessfulRead stamps from stream data. Internal so tests can widen it.
+ internal TimeSpan LivenessStampThrottle { get; set; } = TimeSpan.FromSeconds(1);
+
+ private long _lastLivenessStampTicks;
+
+ ///
+ /// Advance LastSuccessfulRead from inbound stream data, at most once per
+ /// — this runs on the pump's dispatch loop for every
+ /// event, so a 50k-tag galaxy must not pay a health-record allocation per value.
+ ///
+ private void StampLiveness()
+ {
+ var now = DateTime.UtcNow;
+ var last = Volatile.Read(ref _lastLivenessStampTicks);
+ if (now.Ticks - last < LivenessStampThrottle.Ticks) return;
+ Volatile.Write(ref _lastLivenessStampTicks, now.Ticks);
+ _health = _health with { LastSuccessfulRead = now };
}
///
@@ -1503,7 +2084,8 @@ public sealed class GalaxyDriver
_ownedRepositoryClient ??= GalaxyRepositoryClient.Create(
await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false));
return new TracedGalaxyHierarchySource(
- new GatewayGalaxyHierarchySource(_ownedRepositoryClient), _options.MxAccess.ClientName);
+ new GatewayGalaxyHierarchySource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize),
+ _options.MxAccess.ClientName);
}
///
@@ -1517,54 +2099,19 @@ public sealed class GalaxyDriver
public async ValueTask DisposeAsync()
{
if (_disposed) return;
- _disposed = true;
-
- // Synchronous sub-components first — none of these block.
- try { _deployWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher dispose failed"); }
- try { _supervisor?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor dispose failed"); }
- try { _probeWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher dispose failed"); }
- try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); }
-
- // Async sub-components: await each so we don't block a thread-pool thread
- // on a slow shutdown (e.g. EventPump draining its channel, gRPC stream closing).
- EventPump? pump;
- lock (_pumpLock) { pump = _eventPump; _eventPump = null; }
- if (pump is not null)
+ await _lifecycleGate.WaitAsync().ConfigureAwait(false);
+ try
{
- try { await pump.DisposeAsync().ConfigureAwait(false); }
- catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); }
+ if (_disposed) return;
+ _disposed = true;
+ // Same release sequence as ShutdownAsync, plus the injected seams this instance will
+ // never need again. Idempotent: a ShutdownAsync before this finds nothing left to do.
+ await TeardownRuntimeAsync(final: true, CancellationToken.None).ConfigureAwait(false);
}
-
- IGalaxyAlarmFeed? alarmFeed;
- lock (_alarmFeedLock) { alarmFeed = _alarmFeed; _alarmFeed = null; }
- if (alarmFeed is not null)
+ finally
{
- try { await alarmFeed.DisposeAsync().ConfigureAwait(false); }
- catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); }
+ _lifecycleGate.Release();
}
-
- if (_ownedMxSession is not null)
- {
- try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); }
- catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); }
- _ownedMxSession = null;
- }
-
- if (_ownedMxClient is not null)
- {
- try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); }
- catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); }
- _ownedMxClient = null;
- }
-
- if (_ownedRepositoryClient is not null)
- {
- try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); }
- catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); }
- _ownedRepositoryClient = null;
- }
-
- _hierarchySource = null;
}
///
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs
index a4010d07..cfa757b8 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs
@@ -54,12 +54,18 @@ public sealed class GalaxyDriverProbe : IDriverProbe
if (string.IsNullOrWhiteSpace(host) || port <= 0)
return new(false, "Config has no host/port to probe.", null);
+ // The config's own ProbeTimeoutSeconds caps the probe deadline. It can only tighten the
+ // caller's budget, never extend it — the caller's token fires at `timeout` regardless.
+ timeout = EffectiveTimeout(timeout, opts.ProbeTimeoutSeconds);
+ using var probeCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+ probeCts.CancelAfter(timeout);
+
// --- TCP preflight: fast-fail for closed ports / unreachable hosts ---
var sw = Stopwatch.StartNew();
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
- await socket.ConnectAsync(host, port, ct);
+ await socket.ConnectAsync(host, port, probeCts.Token);
}
catch (SocketException ex)
{
@@ -86,10 +92,7 @@ public sealed class GalaxyDriverProbe : IDriverProbe
{
client = GalaxyRepositoryClient.Create(BuildProbeClientOptions(opts.Gateway, timeout));
- using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
- deadlineCts.CancelAfter(timeout);
-
- await client.TestConnectionAsync(deadlineCts.Token).ConfigureAwait(false);
+ await client.TestConnectionAsync(probeCts.Token).ConfigureAwait(false);
sw.Stop();
// No RpcException ⇒ the gateway answered the gRPC call successfully.
@@ -157,6 +160,26 @@ public sealed class GalaxyDriverProbe : IDriverProbe
_ => (false, $"Reachable at {host}:{port} but gateway gRPC handshake failed: {code}"),
};
+ ///
+ /// The deadline the probe actually runs under: the caller's budget, tightened by the
+ /// config's when that is positive and
+ /// shorter. The knob was authored by the AdminUI and parsed by the factory but consumed by
+ /// nothing — the probe always ran on the caller's budget alone. A non-positive caller budget
+ /// is floored to 1 s before the comparison (the same floor
+ /// applies to the gRPC call budget), so the probe runs briefly rather than cancelling before
+ /// it starts; a longer config value never raises it above that floor.
+ ///
+ /// The budget the host handed the probe.
+ /// The config's ProbeTimeoutSeconds; ignored when not positive.
+ /// The effective probe deadline.
+ internal static TimeSpan EffectiveTimeout(TimeSpan callerTimeout, int configuredSeconds)
+ {
+ var caller = callerTimeout > TimeSpan.Zero ? callerTimeout : TimeSpan.FromSeconds(1);
+ if (configuredSeconds <= 0) return caller;
+ var configured = TimeSpan.FromSeconds(configuredSeconds);
+ return configured < caller ? configured : caller;
+ }
+
///
/// Builds the gRPC client options for the probe ping from the gateway config WITHOUT
/// resolving the API-key secret ref (the host owns secret resolution; the probe sends the
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs
index b766e38b..64080d52 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs
@@ -38,12 +38,19 @@ public sealed class PerPlatformProbeWatcher : IDisposable
private readonly ILogger _logger;
private readonly int _bufferedUpdateIntervalMs;
+ // The driver's fan-out registry. The EventPump resolves every inbound event through it, so
+ // the probe handles must be registered there (under SubscriptionRegistry.ProbeSubscriptionId)
+ // or every ScanState event is dropped as "stale after unsubscribe". Null when the watcher is
+ // driven directly (unit tests of the decoding table).
+ private readonly SubscriptionRegistry? _registry;
+
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
private readonly ConcurrentDictionary _itemHandlesByPlatform =
new(StringComparer.OrdinalIgnoreCase);
private readonly Lock _syncLock = new();
- private bool _disposed;
+ // Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop.
+ private volatile bool _disposed;
/// Initializes a new instance of the PerPlatformProbeWatcher class.
/// The Galaxy subscriber for managing probe subscriptions.
@@ -55,9 +62,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable
HostStatusAggregator aggregator,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
+ : this(subscriber, aggregator, registry: null, logger, bufferedUpdateIntervalMs)
+ {
+ }
+
+ ///
+ /// Driver-side ctor: the watcher publishes its probe handles into
+ /// under so the
+ /// fans ScanState events out to .
+ ///
+ /// The Galaxy subscriber for managing probe subscriptions.
+ /// The host status aggregator for tracking platform connectivity.
+ /// The driver's subscription registry; null to keep the handles private.
+ /// Optional logger for diagnostic messages.
+ /// Buffered update interval in milliseconds; must be >= 0.
+ internal PerPlatformProbeWatcher(
+ IGalaxySubscriber subscriber,
+ HostStatusAggregator aggregator,
+ SubscriptionRegistry? registry,
+ ILogger? logger = null,
+ int bufferedUpdateIntervalMs = 0)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
+ _registry = registry;
_logger = logger ?? NullLogger.Instance;
if (bufferedUpdateIntervalMs < 0)
{
@@ -67,6 +95,103 @@ public sealed class PerPlatformProbeWatcher : IDisposable
_bufferedUpdateIntervalMs = bufferedUpdateIntervalMs;
}
+ /// Current platform → probe-handle bindings, as the registry sees them. Test seam.
+ internal IReadOnlyList CurrentBindings => BuildBindings();
+
+ private IReadOnlyList BuildBindings() =>
+ [.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))];
+
+ ///
+ /// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped —
+ /// a stopped watcher must never write its entry back into a registry the teardown cleared).
+ ///
+ private void PublishBindings()
+ {
+ lock (_syncLock) PublishBindingsLocked();
+ }
+
+ private void PublishBindingsLocked()
+ {
+ if (_registry is null || _disposed) return;
+ var bindings = BuildBindings();
+ if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId);
+ else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings);
+ }
+
+ ///
+ /// Re-issue SubscribeBulk for every tracked platform and replace the handles — the
+ /// reconnect replay path, where the gateway session was recreated and every pre-reconnect
+ /// handle is dead. Platforms the gateway now rejects keep a 0 handle (no events expected).
+ ///
+ /// A cancellation token to cancel the operation.
+ /// The number of probe references re-subscribed.
+ public async Task ResubscribeAsync(CancellationToken cancellationToken)
+ {
+ ObjectDisposedException.ThrowIf(_disposed, this);
+ var platforms = _itemHandlesByPlatform.Keys.ToArray();
+ if (platforms.Length == 0) return 0;
+
+ await SubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false);
+ return platforms.Length;
+ }
+
+ private async Task SubscribeAndRecordAsync(IReadOnlyList platforms, CancellationToken cancellationToken)
+ {
+ var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
+ // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
+ // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
+ // changes are rare so a coarser interval is usually fine; deployments that need
+ // tighter health visibility can dial it down through GalaxyDriverOptions.
+ var results = await _subscriber.SubscribeBulkAsync(
+ probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
+
+ var acquired = new (string Platform, int ItemHandle, string? Error)[platforms.Count];
+ for (var i = 0; i < platforms.Count; i++)
+ {
+ var match = results.FirstOrDefault(r => string.Equals(
+ r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
+ acquired[i] = (platforms[i], match is { WasSuccessful: true } ? match.ItemHandle : 0,
+ match?.ErrorMessage ?? "");
+ }
+
+ // Re-check the stop AFTER the await, atomically with the record + publish: a StopAsync that
+ // ran while SubscribeBulk was in flight already cleared the map and the registry entry, so
+ // recording now would resurrect the probe entry into the next runtime's registry.
+ bool stopped;
+ lock (_syncLock)
+ {
+ stopped = _disposed;
+ if (!stopped)
+ {
+ foreach (var (platform, itemHandle, _) in acquired) _itemHandlesByPlatform[platform] = itemHandle;
+ PublishBindingsLocked();
+ }
+ }
+
+ if (stopped)
+ {
+ var orphans = acquired.Where(a => a.ItemHandle > 0).Select(a => a.ItemHandle).ToArray();
+ if (orphans.Length > 0)
+ {
+ try { await _subscriber.UnsubscribeBulkAsync(orphans, CancellationToken.None).ConfigureAwait(false); }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length);
+ }
+ }
+ return;
+ }
+
+ foreach (var (platform, itemHandle, error) in acquired)
+ {
+ if (itemHandle <= 0)
+ {
+ _logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error);
+ }
+ }
+ }
+
/// Snapshot of platform tag names currently watched.
public IReadOnlyCollection WatchedPlatforms => [.. _itemHandlesByPlatform.Keys];
@@ -109,6 +234,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable
if (toRemove.Count > 0)
{
+ // Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe.
+ PublishBindings();
try
{
await _subscriber.UnsubscribeBulkAsync(
@@ -124,30 +251,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable
if (toAdd.Count == 0) return;
- var probeAddresses = toAdd.Select(p => p + ProbeSuffix).ToArray();
- // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
- // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
- // changes are rare so a coarser interval is usually fine; deployments that need
- // tighter health visibility can dial it down through GalaxyDriverOptions.
- var results = await _subscriber.SubscribeBulkAsync(
- probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
-
- for (var i = 0; i < toAdd.Count; i++)
- {
- var platform = toAdd[i];
- var match = results.FirstOrDefault(r => string.Equals(
- r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
-
- var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0;
- _itemHandlesByPlatform[platform] = itemHandle;
-
- if (itemHandle <= 0)
- {
- _logger.LogWarning(
- "PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}",
- platform, match?.ErrorMessage ?? "");
- }
- }
+ await SubscribeAndRecordAsync(toAdd, cancellationToken).ConfigureAwait(false);
}
///
@@ -196,20 +300,32 @@ public sealed class PerPlatformProbeWatcher : IDisposable
};
}
- /// Disposes the probe watcher and unsubscribes all tracked platforms.
- public void Dispose()
+ ///
+ /// Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway
+ /// (best-effort — a transport error is logged and swallowed). The async counterpart of
+ /// ; the driver's ShutdownAsync uses it so the unsubscribe
+ /// round-trip is awaited rather than blocked on. Idempotent.
+ ///
+ /// Bounds the unsubscribe round-trip.
+ /// A task that completes once the handles have been released (or the attempt gave up).
+ public async Task StopAsync(CancellationToken cancellationToken)
{
- if (_disposed) return;
- _disposed = true;
-
- // Best-effort unsubscribe everything we know about. Run synchronously through
- // GetAwaiter().GetResult() since Dispose is sync; transport errors are swallowed.
- var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray();
- _itemHandlesByPlatform.Clear();
+ int[] liveHandles;
+ lock (_syncLock)
+ {
+ if (_disposed) return;
+ _disposed = true;
+ liveHandles = [.. _itemHandlesByPlatform.Values.Where(h => h > 0)];
+ _itemHandlesByPlatform.Clear();
+ _registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId);
+ }
if (liveHandles.Length > 0)
{
- try { _subscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None).GetAwaiter().GetResult(); }
- catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher dispose unsubscribe failed"); }
+ try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); }
}
}
+
+ /// Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip).
+ public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult();
}
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs
index d6e4fb4b..7eb95234 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs
@@ -187,9 +187,11 @@ internal sealed class EventPump : IAsyncDisposable
NotifyStreamFault(cause);
}
}
- catch (OperationCanceledException) when (ct.IsCancellationRequested)
+ catch (Exception) when (ct.IsCancellationRequested)
{
- // Clean shutdown — no log.
+ // Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled)
+ // rather than OperationCanceledException; either way the pump was told to stop, so this
+ // is not a transport fault and must not wake the reconnect supervisor mid-teardown.
}
catch (Exception ex)
{
@@ -283,14 +285,19 @@ internal sealed class EventPump : IAsyncDisposable
EventsDispatched.Add(1, _clientTag);
}
}
- catch (OperationCanceledException) when (ct.IsCancellationRequested)
+ catch (Exception) when (ct.IsCancellationRequested)
{
// Clean shutdown.
}
catch (Exception ex)
{
+ // The read loop is still draining the stream into a channel nobody reads: the pump is
+ // as dead as after a transport fault, but used to raise nothing — health stayed
+ // Healthy and the comms-loss watchdog never tripped (a clean termination evading the
+ // watchdog). Report it as a stream fault so the supervisor recreates the pump.
_logger.LogWarning(ex,
- "Galaxy EventPump dispatch loop ended with an exception — events past this point will be lost until restart.");
+ "Galaxy EventPump dispatch loop ended with an exception — notifying reconnect supervisor so the pump is recreated.");
+ NotifyStreamFault(ex);
}
}
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs
index 0daae292..75550740 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs
@@ -60,6 +60,12 @@ public sealed class GalaxyMxSession : IAsyncDisposable
///
internal Func? OpenAndRegisterOverrideForTests { get; set; }
+ ///
+ /// Test seam — replaces the gateway CloseSession round-trip in
+ /// so the bounded-close behaviour can be exercised against a never-answering fake.
+ ///
+ internal Func? CloseOverrideForTests { get; set; }
+
///
/// Connect the underlying gateway client + open an MXAccess session + register the
/// configured client name. Idempotent — second calls are no-ops while a connect has
@@ -96,7 +102,7 @@ public sealed class GalaxyMxSession : IAsyncDisposable
{
// A partial open (e.g. OpenSession succeeded but Register threw) must not leave a half-open
// handle that blocks the next reconnect. Tear the partial state down so a retry rebuilds cleanly.
- await TeardownSessionAsync().ConfigureAwait(false);
+ await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false);
throw;
}
}
@@ -112,10 +118,23 @@ public sealed class GalaxyMxSession : IAsyncDisposable
public async Task RecreateAsync(MxGatewayClientOptions clientOptions, CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
- await TeardownSessionAsync().ConfigureAwait(false);
+ await TeardownSessionAsync(cancellationToken).ConfigureAwait(false);
await ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false);
}
+ ///
+ /// Close the gateway session and release the owned client under
+ /// . The SDK's DisposeAsync closes with NO token,
+ /// i.e. under the 30 s DefaultCallTimeout — against a blackholed gateway that parks
+ /// the driver's ShutdownAsync (and the host's PostStop) for the full timeout.
+ /// A close that is cancelled or fails is NOT retried through the SDK dispose (which would
+ /// park again); the channel is dropped and the gateway reaps the session on its idle lease.
+ /// The session can be re-opened afterwards (this does not set _disposed).
+ ///
+ /// Bounds the gateway CloseSession round-trip.
+ /// A task that completes once the session and client are released.
+ public Task CloseAsync(CancellationToken cancellationToken) => TeardownSessionAsync(cancellationToken);
+
///
/// Test seam — attach a session opened externally (e.g. against an in-process gw
/// fake). Skips the gateway-client construction so tests can drive the session
@@ -143,24 +162,45 @@ public sealed class GalaxyMxSession : IAsyncDisposable
{
if (_disposed) return;
_disposed = true;
- await TeardownSessionAsync().ConfigureAwait(false);
+ await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false);
}
///
- /// Resets the connect flag and best-effort disposes + nulls the owned session and client.
- /// Shared by (final teardown) and /
- /// the partial-open recovery path (rebuild teardown) — it does NOT set _disposed, so
- /// a torn-down session can be re-opened.
+ /// Resets the connect flag and best-effort closes + nulls the owned session and client.
+ /// Shared by (final teardown), (bounded
+ /// teardown) and / the partial-open recovery path (rebuild
+ /// teardown) — it does NOT set _disposed, so a torn-down session can be re-opened.
///
- private async Task TeardownSessionAsync()
+ private async Task TeardownSessionAsync(CancellationToken cancellationToken)
{
+ var wasConnected = _connected;
_connected = false;
+ // The override models the gateway round-trip, so it runs only when there was a live
+ // session to close — a second teardown (CloseAsync then DisposeAsync) must not close twice.
+ var closeOverride = CloseOverrideForTests;
+ if (closeOverride is not null && wasConnected)
+ {
+ try { await closeOverride(cancellationToken).ConfigureAwait(false); }
+ catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession close failed or timed out (best-effort)"); }
+ }
+
if (_session is not null)
{
- try { await _session.DisposeAsync().ConfigureAwait(false); }
- catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession session dispose failed (best-effort)"); }
+ var session = _session;
_session = null;
+ try
+ {
+ // CloseAsync(ct) is the bounded gateway round-trip; DisposeAsync only re-runs an
+ // un-replied close (with no token), so it is reached only after a successful one.
+ await session.CloseAsync(cancellationToken).ConfigureAwait(false);
+ await session.DisposeAsync().ConfigureAwait(false);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "GalaxyMxSession close failed or timed out (best-effort) — dropping the channel; the gateway reaps the session on its idle lease.");
+ }
}
if (_ownedClient is not null)
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs
index a472f269..957459df 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs
@@ -120,15 +120,87 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
// Bind the per-batch MX write operations behind the internal IMxWriteOps seam (see below) so the
// whole write pipeline is unit-testable without the sealed SDK session (archreview 06/S-1).
var ops = new SessionMxWriteOps(session, serverHandle);
+ return await WriteBatchAsync(ops, writes, securityResolver, cancellationToken).ConfigureAwait(false);
+ }
+ ///
+ /// Upper bound on write lanes in flight at once. A first-touch write is three round-trips
+ /// (AddItem → AdviseSupervisory → Write) and the batch used to run them strictly serially;
+ /// distinct tags are independent, so their lanes now overlap, bounded so a large batch cannot
+ /// flood the gateway's STA pump. Acks stay serial by design (see GalaxyDriver.AcknowledgeAsync).
+ ///
+ internal const int MaxConcurrentWriteLanes = 8;
+
+ /// Test seam: drive the real batch pipeline against a fake .
+ /// The (fake) MX write operations.
+ /// The batch.
+ /// Per-tag classification resolver.
+ /// The cancellation token.
+ /// One result per request, in input order.
+ internal Task> WriteBatchForTestAsync(
+ IMxWriteOps ops, IReadOnlyList writes,
+ Func securityResolver, CancellationToken ct)
+ => WriteBatchAsync(ops, writes, securityResolver, ct);
+
+ ///
+ /// Run a batch as one lane per distinct tag (OrdinalIgnoreCase): writes to the SAME tag
+ /// stay in input order inside their lane, lanes run concurrently up to
+ /// , and results land at their request's index.
+ /// A single-tag batch takes the direct path with no gate.
+ ///
+ private async Task> WriteBatchAsync(
+ IMxWriteOps ops, IReadOnlyList writes,
+ Func securityResolver, CancellationToken ct)
+ {
var results = new WriteResult[writes.Count];
+ if (writes.Count == 0) return results;
+
+ // Group request indices by tag, first-seen order preserved so the lane start order is
+ // deterministic and a same-tag sequence keeps its authored order.
+ var lanes = new Dictionary>(StringComparer.OrdinalIgnoreCase);
+ var laneOrder = new List>();
for (var i = 0; i < writes.Count; i++)
{
- results[i] = await WriteOneAsync(ops, writes[i],
- securityResolver(writes[i].FullReference), cancellationToken)
- .ConfigureAwait(false);
+ if (!lanes.TryGetValue(writes[i].FullReference, out var lane))
+ {
+ lane = [];
+ lanes[writes[i].FullReference] = lane;
+ laneOrder.Add(lane);
+ }
+ lane.Add(i);
}
+
+ if (laneOrder.Count == 1)
+ {
+ await RunLaneAsync(laneOrder[0]).ConfigureAwait(false);
+ return results;
+ }
+
+ using var gate = new SemaphoreSlim(MaxConcurrentWriteLanes, MaxConcurrentWriteLanes);
+ var tasks = new Task[laneOrder.Count];
+ for (var l = 0; l < laneOrder.Count; l++)
+ {
+ var lane = laneOrder[l];
+ tasks[l] = RunGatedAsync(lane);
+ }
+ await Task.WhenAll(tasks).ConfigureAwait(false);
return results;
+
+ async Task RunGatedAsync(List lane)
+ {
+ await gate.WaitAsync(ct).ConfigureAwait(false);
+ try { await RunLaneAsync(lane).ConfigureAwait(false); }
+ finally { gate.Release(); }
+ }
+
+ async Task RunLaneAsync(List lane)
+ {
+ foreach (var i in lane)
+ {
+ results[i] = await WriteOneAsync(ops, writes[i],
+ securityResolver(writes[i].FullReference), ct).ConfigureAwait(false);
+ }
+ }
}
/// Test seam: drive the real single-write pipeline against a fake
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs
index 15022173..7b6c6410 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs
@@ -62,6 +62,16 @@ public sealed class ReconnectSupervisor : IDisposable
private Task? _recoveryLoop;
private CancellationTokenSource? _loopCts;
private bool _disposed;
+ private int _cyclesStarted;
+ private int _cyclesReleased;
+
+ /// Recovery cycles started so far (one per that left Healthy). Test seam.
+ internal int CyclesStarted => Volatile.Read(ref _cyclesStarted);
+
+ /// Recovery cycles whose has been disposed. Equals
+ /// once every loop has exited — a cycle's CTS used to be released only
+ /// if it happened to be the LAST one when ran. Test seam.
+ internal int CyclesReleased => Volatile.Read(ref _cyclesReleased);
/// Fires after every state transition.
public event EventHandler? StateChanged;
@@ -136,8 +146,13 @@ public sealed class ReconnectSupervisor : IDisposable
TransitionLocked(State.TransportLost, cause.Message);
- _loopCts = new CancellationTokenSource();
- _recoveryLoop = Task.Run(() => RecoveryLoopAsync(_loopCts.Token));
+ // Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's
+ // finally). Before, every cycle allocated a fresh CTS and only the last one was ever
+ // disposed — one leaked CancellationTokenSource per outage for the driver's lifetime.
+ var cts = new CancellationTokenSource();
+ _loopCts = cts;
+ Interlocked.Increment(ref _cyclesStarted);
+ _recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts));
}
}
@@ -156,7 +171,26 @@ public sealed class ReconnectSupervisor : IDisposable
}
}
- private async Task RecoveryLoopAsync(CancellationToken ct)
+ private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts)
+ {
+ try
+ {
+ await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false);
+ }
+ finally
+ {
+ // Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may
+ // race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object.
+ lock (_stateLock)
+ {
+ if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null;
+ }
+ cycleCts.Dispose();
+ Interlocked.Increment(ref _cyclesReleased);
+ }
+ }
+
+ private async Task RecoveryLoopCoreAsync(CancellationToken ct)
{
var attempt = 0;
while (!ct.IsCancellationRequested)
@@ -240,8 +274,13 @@ public sealed class ReconnectSupervisor : IDisposable
}
}
- /// Disposes the supervisor and cancels any in-flight recovery loop.
- public void Dispose()
+ ///
+ /// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's
+ /// thread. The async counterpart of ; the driver's ShutdownAsync
+ /// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent.
+ ///
+ /// A task that completes once the recovery loop has exited.
+ public async Task StopAsync()
{
if (_disposed) return;
_disposed = true;
@@ -250,13 +289,18 @@ public sealed class ReconnectSupervisor : IDisposable
Task? loop;
lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }
- cts?.Cancel();
+ // The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
+ // throws on Cancel, and a disposed one is a no-op on Dispose.
+ try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ }
if (loop is not null)
{
- try { loop.GetAwaiter().GetResult(); } catch { /* shutdown */ }
+ try { await loop.ConfigureAwait(false); } catch { /* shutdown */ }
}
cts?.Dispose();
}
+
+ /// Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits).
+ public void Dispose() => StopAsync().GetAwaiter().GetResult();
}
///
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs
index 512099f5..1a6ac2ff 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs
@@ -18,10 +18,39 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
///
internal sealed class SubscriptionRegistry
{
+ ///
+ /// Reserved subscription id for the per-platform ScanState probe handles
+ /// (PerPlatformProbeWatcher). Driver subscription ids are positive and
+ /// monotonic (), so a negative id can never collide.
+ /// The probe handles used to live only in the watcher's own map, so the
+ /// — which resolves subscribers through THIS registry —
+ /// dropped every probe event as "stale after unsubscribe" and per-platform host
+ /// status never updated.
+ ///
+ public const long ProbeSubscriptionId = -1;
+
private readonly ConcurrentDictionary _bySubscriptionId = new();
private readonly ConcurrentDictionary> _subscribersByItemHandle = new();
private long _nextSubscriptionId;
+ /// True when is currently registered.
+ /// The subscription identifier.
+ /// Whether the id is tracked.
+ public bool Contains(long subscriptionId) => _bySubscriptionId.ContainsKey(subscriptionId);
+
+ ///
+ /// when the id is already tracked (so its stale reverse-map entries
+ /// are dropped), otherwise. Used by the probe watcher, whose single
+ /// reserved subscription is re-issued on every platform-set change.
+ ///
+ /// The subscription identifier.
+ /// The tag bindings for the subscription.
+ public void Upsert(long subscriptionId, IReadOnlyList bindings)
+ {
+ if (Contains(subscriptionId)) Rebind(subscriptionId, bindings);
+ else Register(subscriptionId, bindings);
+ }
+
/// Gets the number of tracked subscriptions.
public int TrackedSubscriptionCount => _bySubscriptionId.Count;
/// Gets the number of tracked item handles.
@@ -100,6 +129,17 @@ internal sealed class SubscriptionRegistry
}
+ ///
+ /// Drop every subscription and the whole reverse map. Used by the driver's runtime teardown:
+ /// once the gateway session is closed every item handle in here is dead, and the host
+ /// re-issues its subscriptions (fresh ids) after the next InitializeAsync.
+ ///
+ public void Clear()
+ {
+ _bySubscriptionId.Clear();
+ _subscribersByItemHandle.Clear();
+ }
+
/// Snapshot every active binding for diagnostic output.
/// A flattened list of every tag binding across all tracked subscriptions.
public IReadOnlyList SnapshotAllBindings() =>
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor
index 17468d7d..705f70c7 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor
@@ -92,7 +92,7 @@
-
Default 5000 objects per page.
+
Objects per DiscoverHierarchy page the driver asks the gateway for. Default 5000.
@@ -138,7 +138,7 @@
-
Max 60. Used by Test Connect. Default 30.
+
Caps the Test Connect probe deadline (the probe's own budget still applies, so this only shortens it). Max 60. Default 30.
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs
new file mode 100644
index 00000000..d0767592
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs
@@ -0,0 +1,92 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
+
+///
+/// Review 2026-10-21 G-5: Repository.DiscoverPageSize was authored and parsed but reached
+/// nothing — the client's typed DiscoverHierarchyAsync hard-codes 5000. The source now pages
+/// the raw RPC itself at the configured size.
+///
+public sealed class GatewayGalaxyHierarchySourcePagingTests
+{
+ private static GalaxyObject Obj(int id) => new() { GobjectId = id, TagName = $"Obj{id}" };
+
+ /// Every request carries the configured page size and the page tokens chain until the last page.
+ [Fact]
+ public async Task Pages_at_the_configured_size_and_chains_tokens()
+ {
+ var requests = new List();
+ Task Discover(DiscoverHierarchyRequest req, CancellationToken ct)
+ {
+ requests.Add(req);
+ var reply = new DiscoverHierarchyReply();
+ switch (req.PageToken)
+ {
+ case "":
+ reply.Objects.Add([Obj(1), Obj(2)]);
+ reply.NextPageToken = "p2";
+ break;
+ case "p2":
+ reply.Objects.Add([Obj(3), Obj(4)]);
+ reply.NextPageToken = "p3";
+ break;
+ default:
+ reply.Objects.Add(Obj(5));
+ reply.NextPageToken = "";
+ break;
+ }
+ return Task.FromResult(reply);
+ }
+
+ var objects = await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 2, CancellationToken.None);
+
+ objects.Select(o => o.GobjectId).ShouldBe([1, 2, 3, 4, 5]);
+ requests.Count.ShouldBe(3);
+ requests.ShouldAllBe(r => r.PageSize == 2);
+ requests.Select(r => r.PageToken).ShouldBe(["", "p2", "p3"]);
+ }
+
+ /// A different configured size is what goes on the wire (positive control for the knob).
+ [Fact]
+ public async Task A_different_page_size_is_sent_on_the_wire()
+ {
+ int? sent = null;
+ Task Discover(DiscoverHierarchyRequest req, CancellationToken ct)
+ {
+ sent = req.PageSize;
+ return Task.FromResult(new DiscoverHierarchyReply());
+ }
+
+ await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 250, CancellationToken.None);
+ sent.ShouldBe(250);
+ }
+
+ /// A non-positive configured size falls back to the client library's 5000 default.
+ [Theory]
+ [InlineData(0)]
+ [InlineData(-7)]
+ public void Non_positive_page_size_falls_back_to_the_default(int configured)
+ => GatewayGalaxyHierarchySource.EffectivePageSize(configured).ShouldBe(GatewayGalaxyHierarchySource.DefaultPageSize);
+
+ /// A repeated page token is refused rather than walked forever.
+ [Fact]
+ public async Task A_repeated_page_token_is_refused()
+ {
+ var calls = 0;
+ Task Discover(DiscoverHierarchyRequest req, CancellationToken ct)
+ {
+ calls++;
+ var reply = new DiscoverHierarchyReply { NextPageToken = "same" };
+ reply.Objects.Add(Obj(calls));
+ return Task.FromResult(reply);
+ }
+
+ var ex = await Should.ThrowAsync(
+ () => GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 10, CancellationToken.None));
+ ex.Message.ShouldContain("repeated page token");
+ calls.ShouldBe(2);
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs
new file mode 100644
index 00000000..a385ecfb
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs
@@ -0,0 +1,60 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
+
+/// The platform-only query the driver seeds its ScanState probe watcher from.
+public sealed class GatewayGalaxyPlatformSourceTests
+{
+ private static GalaxyObject Obj(string tag, params string[] chain)
+ {
+ var o = new GalaxyObject { TagName = tag };
+ o.TemplateChain.Add(chain);
+ return o;
+ }
+
+ /// Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered.
+ [Fact]
+ public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered()
+ {
+ var requests = new List();
+ Task Discover(DiscoverHierarchyRequest req, CancellationToken ct)
+ {
+ requests.Add(req);
+ var reply = new DiscoverHierarchyReply();
+ if (req.PageToken == "")
+ {
+ reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]);
+ reply.NextPageToken = "p2";
+ }
+ else
+ {
+ reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]);
+ }
+ return Task.FromResult(reply);
+ }
+
+ var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None);
+
+ names.ShouldBe(["Plat01", "Eng01"]);
+ requests.Count.ShouldBe(2);
+ requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false);
+ requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine"));
+ }
+
+ /// A gateway that strips the template chain is trusted to have filtered server-side.
+ [Fact]
+ public async Task Objects_without_a_template_chain_are_kept()
+ {
+ Task Discover(DiscoverHierarchyRequest req, CancellationToken ct)
+ {
+ var reply = new DiscoverHierarchyReply();
+ reply.Objects.Add(new GalaxyObject { TagName = "Plat01" });
+ return Task.FromResult(reply);
+ }
+
+ (await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 0, CancellationToken.None)).ShouldBe(["Plat01"]);
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs
new file mode 100644
index 00000000..1f5fbb78
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs
@@ -0,0 +1,69 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
+using Shouldly;
+using Xunit;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
+
+///
+/// Review 2026-10-21 G-6: ProbeTimeoutSeconds was authored and compared for equivalence but
+/// never consumed — the probe ran on the caller's budget alone. It now caps the probe deadline
+/// (it can shorten the caller's budget, never extend it).
+///
+[Trait("Category", "Unit")]
+public sealed class GalaxyDriverProbeTimeoutTests
+{
+ /// The effective deadline is the shorter of the caller's budget and the configured cap.
+ [Theory]
+ [InlineData(10, 30, 10)] // caller shorter → caller wins
+ [InlineData(10, 2, 2)] // config shorter → config caps
+ [InlineData(10, 0, 10)] // unset config → caller
+ [InlineData(10, -1, 10)] // nonsense config → caller
+ [InlineData(0, 5, 1)] // no caller budget → 1 s floor beats a longer config
+ [InlineData(0, 0, 1)] // nothing → 1 s floor
+ public void EffectiveTimeout_is_the_shorter_of_caller_and_config(int callerSeconds, int configSeconds, int expectedSeconds)
+ => GalaxyDriverProbe.EffectiveTimeout(TimeSpan.FromSeconds(callerSeconds), configSeconds)
+ .ShouldBe(TimeSpan.FromSeconds(expectedSeconds));
+
+ ///
+ /// Against a port that accepts TCP but never speaks gRPC, a 1 s configured cap returns long
+ /// before the caller's 20 s budget. The gRPC handshake itself does the waiting here, so this
+ /// exercises the real deadline path, not the pure helper.
+ ///
+ [Fact]
+ public async Task Configured_cap_shortens_the_probe_against_a_silent_listener()
+ {
+ var listener = new TcpListener(IPAddress.Loopback, 0);
+ listener.Start();
+ try
+ {
+ var port = ((IPEndPoint)listener.LocalEndpoint).Port;
+ var config = $$"""
+ {
+ "Gateway": { "Endpoint": "http://127.0.0.1:{{port}}", "ApiKeySecretRef": "dev:k", "UseTls": false },
+ "MxAccess": { "ClientName": "probe-timeout" },
+ "Repository": {},
+ "Reconnect": {},
+ "ProbeTimeoutSeconds": 1
+ }
+ """;
+
+ var sw = Stopwatch.StartNew();
+ var result = await new GalaxyDriverProbe().ProbeAsync(config, TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
+ sw.Stop();
+
+ result.Ok.ShouldBeFalse();
+ sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), $"the 1 s configured cap must bound the probe (message: {result.Message})");
+ result.Message.ShouldNotBeNull();
+ (result.Message!.Contains("timed out", StringComparison.OrdinalIgnoreCase)
+ || result.Message.Contains("DeadlineExceeded", StringComparison.OrdinalIgnoreCase)
+ || result.Message.Contains("handshake failed", StringComparison.OrdinalIgnoreCase))
+ .ShouldBeTrue($"unexpected message: '{result.Message}'");
+ }
+ finally
+ {
+ listener.Stop();
+ }
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs
new file mode 100644
index 00000000..511bb696
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs
@@ -0,0 +1,112 @@
+using System.Runtime.CompilerServices;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Health;
+
+///
+/// Review 2026-10-21 G-4: the watcher's probe handles must live in the driver's
+/// under the reserved id, follow the platform set, be
+/// rebound on , and leave on stop.
+///
+public sealed class PerPlatformProbeWatcherRegistryTests
+{
+ private sealed class FakeSubscriber : IGalaxySubscriber
+ {
+ private int _next;
+ public List Unsubscribes { get; } = [];
+
+ public Task> SubscribeBulkAsync(
+ IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+ => Task.FromResult>(
+ [.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = ++_next, WasSuccessful = true })]);
+
+ public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken)
+ {
+ Unsubscribes.Add([.. itemHandles]);
+ return Task.CompletedTask;
+ }
+
+ public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ await Task.Delay(Timeout.Infinite, cancellationToken);
+ yield break;
+ }
+ }
+
+ private static (long Id, string Ref)[] Resolve(SubscriptionRegistry registry, int handle) =>
+ [.. registry.ResolveSubscribers(handle)];
+
+ /// Synced platforms resolve through the registry under the reserved id; removed ones stop resolving.
+ [Fact]
+ public async Task Sync_registers_probe_handles_under_the_reserved_id_and_tracks_removals()
+ {
+ var registry = new SubscriptionRegistry();
+ using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
+
+ await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
+
+ registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue();
+ var bindings = watcher.CurrentBindings;
+ bindings.Count.ShouldBe(2);
+ foreach (var b in bindings)
+ {
+ Resolve(registry, b.ItemHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, b.FullReference)]);
+ }
+
+ var bHandle = bindings.Single(b => b.FullReference == "B.ScanState").ItemHandle;
+ await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
+ Resolve(registry, bHandle).ShouldBeEmpty("a removed platform's handle must leave the fan-out map");
+ var aHandle = bindings.Single(b => b.FullReference == "A.ScanState").ItemHandle;
+ Resolve(registry, aHandle).ShouldHaveSingleItem("positive control: the kept platform still resolves");
+ }
+
+ /// Resubscribe replaces the handles and rebinds: old handles stop resolving, new ones resolve.
+ [Fact]
+ public async Task Resubscribe_rebinds_the_registry_to_the_fresh_handles()
+ {
+ var registry = new SubscriptionRegistry();
+ using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
+ await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
+ var oldHandle = watcher.CurrentBindings.Single().ItemHandle;
+
+ (await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
+
+ var newHandle = watcher.CurrentBindings.Single().ItemHandle;
+ newHandle.ShouldNotBe(oldHandle);
+ Resolve(registry, oldHandle).ShouldBeEmpty();
+ Resolve(registry, newHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, "A.ScanState")]);
+ registry.TrackedSubscriptionCount.ShouldBe(1);
+ }
+
+ /// StopAsync unsubscribes the live handles and removes the reserved entry.
+ [Fact]
+ public async Task StopAsync_unsubscribes_and_removes_the_reserved_entry()
+ {
+ var registry = new SubscriptionRegistry();
+ var subscriber = new FakeSubscriber();
+ var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
+ await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
+ var handles = watcher.CurrentBindings.Select(b => b.ItemHandle).OrderBy(h => h).ToArray();
+
+ await watcher.StopAsync(CancellationToken.None);
+
+ subscriber.Unsubscribes.ShouldHaveSingleItem().OrderBy(h => h).ShouldBe(handles);
+ registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse();
+ registry.TrackedItemHandleCount.ShouldBe(0);
+ await Should.NotThrowAsync(() => watcher.StopAsync(CancellationToken.None)); // idempotent
+ }
+
+ /// A watcher without a registry (the decoding-table unit tests) still works and registers nothing.
+ [Fact]
+ public async Task Without_a_registry_the_watcher_keeps_its_handles_private()
+ {
+ using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator());
+ await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
+ watcher.CurrentBindings.ShouldHaveSingleItem();
+ (await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs
new file mode 100644
index 00000000..9d2f16fe
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs
@@ -0,0 +1,17 @@
+using Xunit;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
+
+///
+/// Classes that assert on process-wide telemetry (an ActivityListener on the Galaxy
+/// ActivitySource, a MeterListener on the Galaxy meter) run with parallelization disabled:
+/// those listeners see every Galaxy span / measurement in the process, so any parallel class that
+/// drives a traced subscriber or the data writer — including a driver's background probe seed that
+/// outlives its test — pollutes their counts (F-FU6, observed flaky in lane F fix round 2).
+///
+[CollectionDefinition(Name, DisableParallelization = true)]
+public sealed class ProcessWideTelemetryCollection
+{
+ /// The collection name.
+ public const string Name = "Process-wide Galaxy telemetry";
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs
new file mode 100644
index 00000000..8d48d6d5
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs
@@ -0,0 +1,96 @@
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Google.Protobuf.WellKnownTypes;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// Review 2026-10-21 G-10: a dispatch-loop exit left the read loop draining into a channel
+/// nobody read, with health Healthy and nothing telling the supervisor — a clean termination
+/// evading the watchdog. The exit now reports a stream fault. The fault is injected through
+/// the pump's handle factory, the one dispatch-side seam that runs outside the per-handler
+/// isolation.
+///
+public sealed class EventPumpDispatchFaultTests
+{
+ private const int WaitMs = 10_000;
+
+ private sealed class ChannelSubscriber : IGalaxySubscriber
+ {
+ private readonly Channel _events = Channel.CreateUnbounded();
+
+ public Task> SubscribeBulkAsync(
+ IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+ => Task.FromResult>([]);
+
+ public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
+
+ public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+ {
+ yield return ev;
+ }
+ }
+
+ public ValueTask EmitAsync(int handle) => _events.Writer.WriteAsync(new MxEvent
+ {
+ Family = MxEventFamily.OnDataChange,
+ ItemHandle = handle,
+ Value = new MxValue { DoubleValue = 1 },
+ Quality = 192,
+ SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+ });
+ }
+
+ /// A dispatch-loop exception reports a stream fault carrying that exception.
+ [Fact]
+ public async Task Dispatch_loop_exit_reports_a_stream_fault()
+ {
+ var subscriber = new ChannelSubscriber();
+ var registry = new SubscriptionRegistry();
+ registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
+ var fault = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ await using var pump = new EventPump(
+ subscriber, registry,
+ handleFactory: _ => throw new InvalidDataException("synthetic dispatch failure"),
+ channelCapacity: 4, clientName: "DispatchFault",
+ onStreamFault: ex => fault.TrySetResult(ex));
+ pump.Start();
+
+ await subscriber.EmitAsync(7);
+
+ (await Task.WhenAny(fault.Task, Task.Delay(WaitMs))).ShouldBe(fault.Task,
+ "a dispatch-loop exit must be reported as a stream fault");
+ (await fault.Task).ShouldBeOfType();
+ }
+
+ /// Positive control: the same event with a working handle factory dispatches and reports no fault.
+ [Fact]
+ public async Task Same_event_with_a_working_factory_dispatches_without_a_fault()
+ {
+ var subscriber = new ChannelSubscriber();
+ var registry = new SubscriptionRegistry();
+ registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
+ var faulted = false;
+ var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ await using var pump = new EventPump(
+ subscriber, registry, channelCapacity: 4, clientName: "DispatchFault",
+ onStreamFault: _ => faulted = true);
+ pump.OnDataChange += (_, e) => delivered.TrySetResult(e);
+ pump.Start();
+
+ await subscriber.EmitAsync(7);
+
+ (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task);
+ (await delivered.Task).FullReference.ShouldBe("Tank01.Level");
+ faulted.ShouldBeFalse();
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs
new file mode 100644
index 00000000..9876c09f
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs
@@ -0,0 +1,406 @@
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Google.Protobuf.WellKnownTypes;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// Lane F fix round 1 (review of #735): the probe watcher is seeded at InitializeAsync
+/// and re-seeded on rediscovery (the runtime never calls DiscoverAsync on a live driver);
+/// a SubscribeAsync landing mid-teardown cannot create a pump on the dying subscriber; a
+/// gateway close that never answers cannot park ShutdownAsync; a re-subscribe that
+/// straddles a teardown never rebinds into the next runtime's registry.
+///
+public sealed class GalaxyDriverFixRound1Tests
+{
+ internal const int WaitMs = 10_000;
+
+ internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new(
+ new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
+ new GalaxyMxAccessOptions("FixRound1"),
+ new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
+ new GalaxyReconnectOptions());
+
+ // ───────────────────────────── fakes ─────────────────────────────
+
+ private sealed class ScriptedPlatformSource : IGalaxyPlatformSource
+ {
+ private int _calls;
+ public int Calls => Volatile.Read(ref _calls);
+ public Func> Script { get; init; } = _ => ["Plat01"];
+
+ public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+ => Task.FromResult(Script(Interlocked.Increment(ref _calls)));
+ }
+
+ internal sealed class ParkableSubscriber : IGalaxySubscriber
+ {
+ private readonly Channel _events = Channel.CreateUnbounded();
+ private int _nextHandle;
+ private int _subscribeCalls;
+
+ public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
+ public List Subscribed { get; } = [];
+ public int SubscribeCalls => Volatile.Read(ref _subscribeCalls);
+
+ /// When set, UnsubscribeBulk parks until .
+ public bool ParkUnsubscribe { get; set; }
+ public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult();
+
+ /// When > 0, that SubscribeBulk call (1-based) parks until .
+ public int ParkSubscribeCall { get; set; }
+ public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult();
+
+ public async Task> SubscribeBulkAsync(
+ IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+ {
+ var call = Interlocked.Increment(ref _subscribeCalls);
+ if (call == ParkSubscribeCall)
+ {
+ SubscribeEntered.TrySetResult();
+ await _releaseSubscribe.Task.WaitAsync(cancellationToken);
+ }
+ var results = new List();
+ lock (Subscribed)
+ {
+ foreach (var r in fullReferences)
+ {
+ var h = Interlocked.Increment(ref _nextHandle);
+ Subscribed.Add(r);
+ HandleByRef[r] = h;
+ results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
+ }
+ }
+ return results;
+ }
+
+ /// Every handle passed to UnsubscribeBulk, in call order.
+ public List Unsubscribed { get; } = [];
+
+ public async Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken)
+ {
+ lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
+ if (ParkUnsubscribe)
+ {
+ UnsubscribeEntered.TrySetResult();
+ await _releaseUnsubscribe.Task.WaitAsync(cancellationToken);
+ }
+ }
+
+ public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+ {
+ yield return ev;
+ }
+ }
+
+ public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
+ {
+ Family = MxEventFamily.OnDataChange,
+ ItemHandle = handle,
+ Value = new MxValue { BoolValue = value },
+ Quality = 192,
+ SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+ });
+ }
+
+ internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource
+ {
+ public Channel Events { get; } = Channel.CreateUnbounded();
+ public int Opened;
+
+ public async IAsyncEnumerable WatchAsync(
+ DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ Interlocked.Increment(ref Opened);
+ await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+ {
+ yield return ev;
+ }
+ }
+
+ public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent
+ {
+ Sequence = sequence,
+ TimeOfLastDeployPresent = true,
+ TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
+ });
+ }
+
+ internal static async Task WaitUntilAsync(Func condition, string because)
+ {
+ var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
+ while (!condition())
+ {
+ if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
+ await Task.Delay(10);
+ }
+ }
+
+ internal static HostState? StateOf(GalaxyDriver driver, string host) =>
+ driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
+
+ // ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ─────────────────────────────
+
+ /// A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses.
+ [Fact]
+ public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new ScriptedPlatformSource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ // The seed runs in the background (fix round 2, N1) — init does not wait on it.
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
+ "the probe watcher must be seeded after init");
+ platforms.Calls.ShouldBe(1);
+ lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState");
+ driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
+
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
+ await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
+ await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
+ "a ScanState event must surface the seeded platform on GetHostStatuses");
+ }
+
+ /// A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds.
+ [Fact]
+ public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds()
+ {
+ var sub = new ParkableSubscriber();
+ var deploys = new ChannelDeployWatchSource();
+ var platforms = new ScriptedPlatformSource
+ {
+ Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"],
+ };
+ await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
+ subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
+ var raised = 0;
+ driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init");
+ await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run");
+ sub.Subscribed.ShouldBeEmpty();
+
+ await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
+ await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed)
+ await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy
+
+ await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery");
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
+ "the rediscovery must re-seed the probe set");
+ platforms.Calls.ShouldBe(2);
+ sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true);
+ }
+
+ private sealed class ParkingPlatformSource : IGalaxyPlatformSource
+ {
+ private int _calls;
+ public int Calls => Volatile.Read(ref _calls);
+ public int ParkCall { get; init; }
+ public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public void Release() => _release.TrySetResult();
+
+ public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+ {
+ var call = Interlocked.Increment(ref _calls);
+ if (call == ParkCall)
+ {
+ Entered.TrySetResult();
+ await _release.Task.WaitAsync(cancellationToken);
+ return ["Plat01"]; // the pre-deploy view this pass fetched
+ }
+ return call == 1 ? [] : ["Plat01", "Eng02"];
+ }
+ }
+
+ ///
+ /// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts
+ /// after it — otherwise the probe set would stay on the stale view until the next deploy.
+ ///
+ [Fact]
+ public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass()
+ {
+ var sub = new ParkableSubscriber();
+ var deploys = new ChannelDeployWatchSource();
+ var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery
+ await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
+ subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
+ var raised = 0;
+ driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
+ try
+ {
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
+ await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap
+ await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
+ (await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task,
+ "the first rediscovery pass must be parked mid-fetch");
+
+ await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero));
+ await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery");
+ }
+ finally
+ {
+ platforms.Release();
+ }
+
+ await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed");
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
+ "the final probe set must reflect the latest deploy, not the parked pass's stale view");
+ }
+
+ /// The production path (owned runtime) queries the platform source at init, with no DiscoverAsync.
+ [Fact]
+ public async Task Production_path_queries_the_platform_source_at_init()
+ {
+ var platforms = new ScriptedPlatformSource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms)
+ {
+ SessionOpenOverrideForTests = _ => Task.CompletedTask,
+ };
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init");
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+ }
+
+ // ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ─────────────────────────────
+
+ ///
+ /// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before
+ /// the subscriber is released) is refused and leaves no pump; before the shutdown the same
+ /// call works (positive control).
+ ///
+ [Fact]
+ public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
+ "the (background) seed must give the probe watcher its handle first");
+
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+ driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
+
+ sub.ParkUnsubscribe = true;
+ var shutdown = driver.ShutdownAsync(CancellationToken.None);
+ var subscribeCallsBefore = sub.SubscribeCalls;
+ try
+ {
+ (await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task,
+ "the teardown must be parked inside the probe watcher's unsubscribe");
+ shutdown.IsCompleted.ShouldBeFalse();
+
+ // The pump is still live here (the teardown takes it after the probe watcher) — reusing it
+ // is exactly the race: the call must be refused, not served on the dying subscriber.
+ driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken");
+ await Should.ThrowAsync(
+ () => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None));
+ sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway");
+ }
+ finally
+ {
+ // Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs.
+ sub.ReleaseUnsubscribe();
+ }
+ await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+ driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown");
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
+ }
+
+ // ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
+
+ /// A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline.
+ [Fact]
+ public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers()
+ {
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource())
+ {
+ SessionOpenOverrideForTests = _ => Task.CompletedTask,
+ SessionCloseDeadline = TimeSpan.FromMilliseconds(300),
+ };
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var closeToken = CancellationToken.None;
+ driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct =>
+ {
+ closeToken = ct;
+ closeEntered.TrySetResult();
+ await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway
+ };
+
+ var sw = System.Diagnostics.Stopwatch.StartNew();
+ await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+ sw.Stop();
+
+ closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted");
+ closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline");
+ sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s");
+ driver.OwnedMxSessionForTests.ShouldBeNull();
+ driver.GetHealth().State.ShouldBe(DriverState.Unknown);
+ }
+
+ // ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ─────────────────────────────
+
+ ///
+ /// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared
+ /// registry; the same replay without a teardown rebinds (positive control).
+ ///
+ [Fact]
+ public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind()
+ {
+ // Positive control first.
+ {
+ var sub = new ParkableSubscriber();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+ await driver.InvokeReplayForTestAsync(CancellationToken.None);
+ driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription");
+ sub.SubscribeCalls.ShouldBe(2);
+ }
+
+ {
+ var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+
+ var replay = driver.InvokeReplayForTestAsync(CancellationToken.None);
+ (await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task,
+ "the replay must be parked inside its SubscribeBulk");
+
+ await driver.ShutdownAsync(CancellationToken.None);
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0);
+
+ sub.ReleaseSubscribe();
+ await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles");
+ }
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs
new file mode 100644
index 00000000..ddd97023
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs
@@ -0,0 +1,313 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// Lane F fix round 2 (re-review of #735):
+///
+/// N1 — the init platform seed runs in the background: a slow or cancelled repository
+/// fetch can neither delay nor fail InitializeAsync (Galaxy's Tier-A Initialize budget is
+/// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown
+/// is discarded.
+/// N2 — a seed requested under a new runtime always gets a pass of its own, even when a
+/// worker from the previous runtime is still running.
+/// N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails
+/// and unsubscribes instead of registering into the next runtime's registry.
+/// The probe watcher's own subscribe re-checks its stop after the await.
+///
+///
+public sealed class GalaxyDriverFixRound2Tests
+{
+ private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2);
+
+ /// A platform source whose fetches park until released (optionally ignoring cancellation).
+ private sealed class GatedPlatformSource : IGalaxyPlatformSource
+ {
+ private int _calls;
+ private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public int Calls => Volatile.Read(ref _calls);
+ public int ParkCall { get; init; } = 1;
+ public bool IgnoreCancellation { get; init; }
+ public Func> Result { get; init; } = _ => ["Plat01"];
+ public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public TaskCompletionSource ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public void Release() => _release.TrySetResult();
+
+ public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+ {
+ var call = Interlocked.Increment(ref _calls);
+ if (call == ParkCall)
+ {
+ ParkedToken.TrySetResult(cancellationToken);
+ Entered.TrySetResult();
+ if (IgnoreCancellation) await _release.Task;
+ else await _release.Task.WaitAsync(cancellationToken);
+ }
+ return Result(call);
+ }
+ }
+
+ private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource
+ {
+ private int _calls;
+ public int Calls => Volatile.Read(ref _calls);
+
+ public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+ {
+ Interlocked.Increment(ref _calls);
+ return Task.FromException>(ex);
+ }
+ }
+
+ private static async Task Entered(TaskCompletionSource tcs, string because) =>
+ (await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because);
+
+ // ───────────────────────────── N1: background init seed ─────────────────────────────
+
+ ///
+ /// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy
+ /// inside its budget, and the teardown cancels the parked fetch.
+ ///
+ [Fact]
+ public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new GatedPlatformSource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+ try
+ {
+ var sw = System.Diagnostics.Stopwatch.StartNew();
+ await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
+ sw.Stop();
+
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+ await Entered(platforms.Entered, "positive control: the seed fetch was started");
+ sw.Elapsed.ShouldBeLessThan(InitBudget);
+
+ await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+ var token = await platforms.ParkedToken.Task;
+ token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight");
+ }
+ finally
+ {
+ platforms.Release();
+ }
+ }
+
+ /// A fetch that is cancelled (the gateway's own deadline) does not fail init.
+ [Fact]
+ public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline"));
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+
+ await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
+
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+ await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran");
+ driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
+
+ // The driver keeps serving after the failed seed.
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+ driver.TrackedSubscriptionCountForTests.ShouldBe(1);
+ }
+
+ ///
+ /// The production failure mode: the host's Tier-A Initialize timeout cancels the init token
+ /// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and
+ /// the seed — tied to the runtime, not to the init call — must survive that cancellation and
+ /// still land.
+ ///
+ [Fact]
+ public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new GatedPlatformSource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+ using var initCts = new CancellationTokenSource();
+ try
+ {
+ initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk
+ await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget);
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+
+ await Entered(platforms.Entered, "the seed fetch must be in flight");
+ await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked");
+ (await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse(
+ "the seed belongs to the runtime; the init call's timeout must not cancel it");
+ }
+ finally
+ {
+ platforms.Release();
+ }
+
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
+ "the seed must still land after the init token was cancelled");
+ }
+
+ /// Positive control: a fetch that completes after init still populates host statuses.
+ [Fact]
+ public async Task A_fetch_completing_after_init_populates_host_statuses()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new GatedPlatformSource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+ try
+ {
+ await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
+ await Entered(platforms.Entered, "the seed must be parked");
+ driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
+ }
+ finally
+ {
+ platforms.Release();
+ }
+
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
+ "the late seed must reach the probe watcher");
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
+ await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
+ await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
+ "a ScanState event must surface the late-seeded platform");
+ }
+
+ // ───────────────────────────── N1 generation + N2: straddling worker ─────────────────────────────
+
+ ///
+ /// A seed from the previous runtime that completes after a teardown + re-init is discarded,
+ /// and the new runtime's own seed request still gets its own pass.
+ ///
+ [Fact]
+ public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded()
+ {
+ var sub = new ParkableSubscriber();
+ var platforms = new GatedPlatformSource
+ {
+ IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it
+ Result = call => call == 1 ? ["Stale01"] : ["Plat01"],
+ };
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
+ try
+ {
+ await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
+ await Entered(platforms.Entered, "runtime 1's seed must be parked");
+
+ await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+ await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
+ }
+ finally
+ {
+ platforms.Release();
+ }
+
+ await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass");
+ await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
+ "the new runtime must be seeded");
+ driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]);
+ lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded");
+ }
+
+ // ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ─────────────────────────────
+
+ ///
+ /// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry
+ /// back into the registry, and must release the handles it just acquired. Positive control:
+ /// the same sync without a stop registers.
+ ///
+ [Fact]
+ public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry()
+ {
+ {
+ var registry = new SubscriptionRegistry();
+ using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry);
+ await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
+ registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control");
+ }
+
+ {
+ var registry = new SubscriptionRegistry();
+ var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
+ var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry);
+
+ var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
+ await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk");
+ await watcher.StopAsync(CancellationToken.None);
+
+ sub.ReleaseSubscribe();
+ await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+
+ registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register");
+ watcher.WatchedPlatforms.ShouldBeEmpty();
+ lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"],
+ "the handle acquired after the stop must be released");
+ }
+ }
+
+ // ───────────────────────────── N3: subscribe / read straddling a teardown ─────────────────────────────
+
+ ///
+ /// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing
+ /// and unsubscribes its handles; a subscribe on the live runtime registers (positive control).
+ ///
+ [Fact]
+ public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing()
+ {
+ var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+ driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers");
+
+ var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None);
+ try
+ {
+ await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk");
+ await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0);
+ }
+ finally
+ {
+ sub.ReleaseSubscribe();
+ }
+
+ await Should.ThrowAsync(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry");
+ lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry");
+ }
+
+ /// The subscribe-once read path: same straddle, same outcome.
+ [Fact]
+ public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing()
+ {
+ var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None);
+ try
+ {
+ await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk");
+ await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+ }
+ finally
+ {
+ sub.ReleaseSubscribe();
+ }
+
+ await Should.ThrowAsync(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0);
+ lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]);
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs
new file mode 100644
index 00000000..af842c93
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs
@@ -0,0 +1,217 @@
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Google.Protobuf.WellKnownTypes;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// Review 2026-10-21 G-4 / G-7 / G-11: the per-platform ScanState probing was structurally
+/// dead (its handles were never in the registry the pump resolves through, and a reconnect
+/// never re-subscribed them), discovery fetched the full galaxy twice, and
+/// LastSuccessfulRead never moved on data. These drive the real driver → pump → watcher
+/// → aggregator path with a fake subscriber and a fake hierarchy.
+///
+public sealed class GalaxyDriverProbeWiringTests
+{
+ private const int WaitMs = 10_000;
+
+ private static GalaxyDriverOptions Opts() => new(
+ new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
+ new GalaxyMxAccessOptions("ProbeWiring"),
+ new GalaxyRepositoryOptions(WatchDeployEvents: false),
+ new GalaxyReconnectOptions());
+
+ private sealed class CountingHierarchySource : IGalaxyHierarchySource
+ {
+ public int Calls;
+
+ public Task> GetHierarchyAsync(CancellationToken cancellationToken)
+ {
+ Interlocked.Increment(ref Calls);
+ var platform = new GalaxyObject { GobjectId = 1, TagName = "Plat01", ContainedName = "Plat01" };
+ platform.TemplateChain.Add("$WinPlatform");
+ var tank = new GalaxyObject { GobjectId = 2, TagName = "Tank01", ContainedName = "Tank01", ParentGobjectId = 1 };
+ tank.TemplateChain.Add("$UserDefined");
+ tank.Attributes.Add(new GalaxyAttribute { AttributeName = "Level", FullTagReference = "Tank01.Level", MxDataType = 5 });
+ return Task.FromResult>([platform, tank]);
+ }
+ }
+
+ private sealed class StreamingSubscriber : IGalaxySubscriber
+ {
+ private readonly Channel _events = Channel.CreateUnbounded();
+ private int _nextHandle;
+
+ public List Subscribed { get; } = [];
+ public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
+
+ public Task> SubscribeBulkAsync(
+ IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+ {
+ var results = new List();
+ lock (Subscribed)
+ {
+ foreach (var r in fullReferences)
+ {
+ var h = Interlocked.Increment(ref _nextHandle);
+ Subscribed.Add(r);
+ HandleByRef[r] = h;
+ results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
+ }
+ }
+ return Task.FromResult>(results);
+ }
+
+ public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
+
+ public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+ {
+ yield return ev;
+ }
+ }
+
+ public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
+ {
+ Family = MxEventFamily.OnDataChange,
+ ItemHandle = handle,
+ Value = new MxValue { BoolValue = value },
+ Quality = 192,
+ SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+ });
+
+ public ValueTask EmitDoubleAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
+ {
+ Family = MxEventFamily.OnDataChange,
+ ItemHandle = handle,
+ Value = new MxValue { DoubleValue = value },
+ Quality = 192,
+ SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+ });
+ }
+
+ private sealed class NullBuilder : IAddressSpaceBuilder
+ {
+ public IAddressSpaceBuilder Folder(string browseName, string displayName) => this;
+ public IVariableHandle Variable(string browseName, string displayName, DriverAttributeInfo attributeInfo) => new Handle(attributeInfo.FullName);
+ public void AddProperty(string browseName, DriverDataType dataType, object? value) { }
+ private sealed class Handle(string fullReference) : IVariableHandle
+ {
+ public string FullReference { get; } = fullReference;
+ public IAlarmConditionSink MarkAsAlarmCondition(AlarmConditionInfo alarmInfo) => new NoopSink();
+ }
+ private sealed class NoopSink : IAlarmConditionSink
+ {
+ public void OnTransition(AlarmEventArgs args) { }
+ }
+ }
+
+ private static async Task WaitUntilAsync(Func condition, string because)
+ {
+ var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
+ while (!condition())
+ {
+ if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
+ await Task.Delay(10);
+ }
+ }
+
+ private static HostState? StateOf(GalaxyDriver driver, string host) =>
+ driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
+
+ ///
+ /// A ScanState event on the probe handle updates GetHostStatuses; after a reconnect replay
+ /// the probe is re-subscribed and events on the NEW handle keep updating it, while the dead
+ /// pre-reconnect handle no longer does. Probe traffic never reaches public OnDataChange.
+ ///
+ [Fact]
+ public async Task ScanState_event_updates_GetHostStatuses_and_survives_a_reconnect()
+ {
+ var sub = new StreamingSubscriber();
+ var source = new CountingHierarchySource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
+ var publicRefs = new List();
+ driver.OnDataChange += (_, e) => { lock (publicRefs) publicRefs.Add(e.FullReference); };
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
+ // The pump starts on the first value subscription (the probe watcher does not start it).
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+
+ var probeHandle = sub.HandleByRef["Plat01.ScanState"];
+ await sub.EmitBoolAsync(probeHandle, true);
+ await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
+ "a ScanState=true event must surface Plat01 as Running");
+
+ // Reconnect replay: every pre-reconnect handle is dead; the probe must rebind.
+ await driver.InvokeReplayForTestAsync(CancellationToken.None);
+ var rebound = sub.HandleByRef["Plat01.ScanState"];
+ rebound.ShouldNotBe(probeHandle, "replay must re-subscribe the probe and record the fresh handle");
+
+ await sub.EmitBoolAsync(rebound, false);
+ await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Stopped,
+ "a ScanState=false event on the post-reconnect handle must surface Plat01 as Stopped");
+
+ // The dead handle is no longer routed (positive control: the live one still is).
+ await sub.EmitBoolAsync(probeHandle, true);
+ await sub.EmitDoubleAsync(sub.HandleByRef["Tank01.Level"], 1.0);
+ await WaitUntilAsync(() => { lock (publicRefs) return publicRefs.Contains("Tank01.Level"); }, "value traffic must still flow");
+ StateOf(driver, "Plat01").ShouldBe(HostState.Stopped, "an event on the pre-reconnect handle must be dropped");
+
+ lock (publicRefs) publicRefs.ShouldNotContain("Plat01.ScanState", "probe traffic is the watcher's, never the host's");
+ }
+
+ /// Discovery asks the gateway for the hierarchy once; the probe set is selected from that same sweep.
+ [Fact]
+ public async Task DiscoverAsync_fetches_the_hierarchy_once_and_still_syncs_the_probe_set()
+ {
+ var sub = new StreamingSubscriber();
+ var source = new CountingHierarchySource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
+
+ source.Calls.ShouldBe(1, "the probe selection must reuse the discoverer's sweep");
+ sub.Subscribed.ShouldContain("Plat01.ScanState", "positive control: the platform probe was still synced");
+ driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
+ }
+
+ /// Inbound stream data advances LastSuccessfulRead (throttled); the throttle holds the stamp still.
+ [Fact]
+ public async Task Stream_data_advances_LastSuccessfulRead_under_a_throttle()
+ {
+ var sub = new StreamingSubscriber();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
+ {
+ LivenessStampThrottle = TimeSpan.FromHours(1),
+ };
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ var initStamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
+ await Task.Delay(20);
+
+ var delivered = 0;
+ driver.OnDataChange += (_, _) => Interlocked.Increment(ref delivered);
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+ var handle = sub.HandleByRef["Tank01.Level"];
+
+ await sub.EmitDoubleAsync(handle, 1.0);
+ await WaitUntilAsync(() => Volatile.Read(ref delivered) == 1, "first event must be delivered");
+ var first = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
+ first.ShouldBeGreaterThan(initStamp, "data on the stream is liveness evidence");
+
+ await Task.Delay(20);
+ await sub.EmitDoubleAsync(handle, 2.0);
+ await WaitUntilAsync(() => Volatile.Read(ref delivered) == 2, "second event must be delivered");
+ driver.GetHealth().LastSuccessfulRead.ShouldBe(first, "inside the throttle window the stamp holds still");
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs
new file mode 100644
index 00000000..cdb2ec69
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs
@@ -0,0 +1,386 @@
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Google.Protobuf.WellKnownTypes;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// used to be a no-op (review 2026-10-21, G-1/G-2/G-9):
+/// the gRPC runtime outlived the driver on every respawn, removal and /raw browse, and
+/// the host's comms-loss recovery (ShutdownAsync → InitializeAsync on the SAME
+/// object) neither tore down nor rebuilt anything. These tests pin the teardown at every
+/// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed,
+/// the owned channels + session — and the rebuild on the next InitializeAsync.
+///
+public sealed class GalaxyDriverShutdownTests
+{
+ private const int WaitMs = 10_000;
+
+ private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new(
+ new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
+ new GalaxyMxAccessOptions("ShutdownTest"),
+ new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
+ new GalaxyReconnectOptions());
+
+ // ───────────────────────────── fakes ─────────────────────────────
+
+ ///
+ /// Subscriber whose event stream records the token it was opened with and completes a
+ /// signal when that token fires — the observable for "the StreamEvents stream was cancelled".
+ ///
+ private sealed class StreamingSubscriber : IGalaxySubscriber
+ {
+ private readonly Channel _events = Channel.CreateUnbounded();
+ private int _nextHandle;
+
+ public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
+ public List Unsubscribed { get; } = [];
+ public int StreamsOpened;
+ public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public Task> SubscribeBulkAsync(
+ IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+ {
+ var results = new List();
+ foreach (var r in fullReferences)
+ {
+ var h = Interlocked.Increment(ref _nextHandle);
+ HandleByRef[r] = h;
+ results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
+ }
+ return Task.FromResult>(results);
+ }
+
+ public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken)
+ {
+ lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
+ return Task.CompletedTask;
+ }
+
+ public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ Interlocked.Increment(ref StreamsOpened);
+ using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult());
+ await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+ {
+ yield return ev;
+ }
+ }
+
+ public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
+ {
+ Family = MxEventFamily.OnDataChange,
+ ItemHandle = handle,
+ Value = new MxValue { DoubleValue = value },
+ Quality = 192,
+ SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+ });
+ }
+
+ ///
+ /// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when
+ /// the enumerator is cancelled or completes) — the observable for "exactly one watcher".
+ ///
+ private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource
+ {
+ private int _open;
+ private int _opened;
+ public Channel Current { get; private set; } = Channel.CreateUnbounded();
+
+ public int Open => Volatile.Read(ref _open);
+ public int Opened => Volatile.Read(ref _opened);
+
+ public async IAsyncEnumerable WatchAsync(
+ DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
+ {
+ Interlocked.Increment(ref _open);
+ Interlocked.Increment(ref _opened);
+ var channel = Current = Channel.CreateUnbounded();
+ try
+ {
+ await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+ {
+ yield return ev;
+ }
+ }
+ finally
+ {
+ Interlocked.Decrement(ref _open);
+ }
+ }
+
+ public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent
+ {
+ Sequence = sequence,
+ TimeOfLastDeployPresent = true,
+ TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
+ });
+ }
+
+ /// Canned platform set so a production-path init never dials the (fake) gateway for one.
+ private sealed class NoPlatforms : IGalaxyPlatformSource
+ {
+ public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+ => Task.FromResult>([]);
+ }
+
+ private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed
+ {
+ public int Starts;
+ public int Disposes;
+ public event EventHandler? OnAlarmTransition;
+ public void Start() => Starts++;
+ public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition(
+ "Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise,
+ AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi"));
+ public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; }
+ }
+
+ private static async Task WaitUntilAsync(Func condition, string because)
+ {
+ var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
+ while (!condition())
+ {
+ if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
+ await Task.Delay(10);
+ }
+ }
+
+ // ───────────────────────────── G-1: streams + pump ─────────────────────────────
+
+ ///
+ /// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the
+ /// registry is empty. Positive control: an event delivered through that same stream
+ /// reached OnDataChange before the shutdown.
+ ///
+ [Fact]
+ public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump()
+ {
+ var sub = new StreamingSubscriber();
+ await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ driver.OnDataChange += (_, e) => delivered.TrySetResult(e);
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+ driver.HasEventPumpForTests.ShouldBeTrue();
+
+ await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0);
+ (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task,
+ "positive control: the stream must deliver before the shutdown");
+
+ await driver.ShutdownAsync(CancellationToken.None);
+
+ // Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the
+ // enumerator's cancellation registration has run by the time it returns (observed flaky).
+ (await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task,
+ "ShutdownAsync must cancel the StreamEvents stream");
+ driver.HasEventPumpForTests.ShouldBeFalse();
+ driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init");
+ driver.GetHealth().State.ShouldBe(DriverState.Unknown);
+ driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown");
+ }
+
+ ///
+ /// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT
+ /// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive
+ /// control on both sides of the shutdown.
+ ///
+ [Fact]
+ public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it()
+ {
+ var feed = new RecordingAlarmFeed();
+ var sub = new StreamingSubscriber();
+ await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
+ subscriber: sub, alarmFeed: feed);
+ var received = 0;
+ driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
+ feed.Emit();
+ received.ShouldBe(1, "positive control: the feed is wired before the shutdown");
+
+ await driver.ShutdownAsync(CancellationToken.None);
+ feed.Emit();
+ received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent");
+ feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector");
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
+ feed.Emit();
+ received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed");
+ feed.Starts.ShouldBe(2);
+ }
+
+ // ───────────────────────────── G-2: one DeployWatcher ─────────────────────────────
+
+ ///
+ /// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch
+ /// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly
+ /// once — five stacked watchers would raise five times.
+ ///
+ [Fact]
+ public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher()
+ {
+ var source = new CountingDeployWatchSource();
+ var sub = new StreamingSubscriber();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
+ deployWatchSource: source);
+ var raised = 0;
+ driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ for (var cycle = 0; cycle < 5; cycle++)
+ {
+ // Let this cycle's watch stream actually open before the next shutdown, so every cycle
+ // holds a live stream the teardown has to cancel (a loop cancelled before its first
+ // open would make the count vacuous).
+ await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open");
+ await driver.ShutdownAsync(CancellationToken.None);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ }
+
+ await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1,
+ $"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}");
+ driver.DeployWatcherForTests.ShouldNotBeNull();
+
+ // Bootstrap (suppressed) then a genuine change.
+ await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
+ await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
+ await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery");
+ await Task.Delay(200);
+ Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy");
+
+ await driver.ShutdownAsync(CancellationToken.None);
+ await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream");
+ driver.DeployWatcherForTests.ShouldBeNull();
+ }
+
+ /// A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one.
+ [Fact]
+ public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher()
+ {
+ var source = new CountingDeployWatchSource();
+ var sub = new StreamingSubscriber();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
+ deployWatchSource: source);
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ var first = driver.DeployWatcherForTests;
+ await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open");
+ await driver.InitializeAsync("{}", CancellationToken.None);
+
+ await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1,
+ $"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}");
+ driver.DeployWatcherForTests.ShouldNotBeSameAs(first);
+ }
+
+ // ───────────────────────────── G-1/G-9: owned runtime, channels + session ─────────────────────────────
+
+ ///
+ /// The production path (no injected seams): InitializeAsync builds the owned client, session
+ /// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the
+ /// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's
+ /// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC
+ /// channels are real but lazy (nothing dials until a call is made).
+ ///
+ [Fact]
+ public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds()
+ {
+ var opens = 0;
+ var source = new CountingDeployWatchSource();
+ await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source,
+ platformSource: new NoPlatforms())
+ {
+ SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; },
+ };
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ opens.ShouldBe(1);
+ var client = driver.OwnedMxClientForTests.ShouldNotBeNull();
+ var session = driver.OwnedMxSessionForTests.ShouldNotBeNull();
+ session.IsConnected.ShouldBeTrue();
+ driver.SupervisorForTests.ShouldNotBeNull();
+ driver.ProbeWatcherForTests.ShouldNotBeNull();
+ driver.DeployWatcherForTests.ShouldNotBeNull();
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+ var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
+
+ await driver.ShutdownAsync(CancellationToken.None);
+
+ session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session");
+ await Should.ThrowAsync(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None),
+ "the owned gateway channel must be disposed");
+ driver.OwnedMxClientForTests.ShouldBeNull();
+ driver.OwnedMxSessionForTests.ShouldBeNull();
+ driver.SupervisorForTests.ShouldBeNull();
+ driver.ProbeWatcherForTests.ShouldBeNull();
+ driver.DeployWatcherForTests.ShouldBeNull();
+ await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled");
+ driver.GetHealth().State.ShouldBe(DriverState.Unknown);
+ driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp);
+
+ // The watchdog's second half: a fresh runtime on the same object.
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one");
+ driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client);
+ driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session);
+ driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue();
+ driver.SupervisorForTests.ShouldNotBeNull();
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+ await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher");
+ }
+
+ ///
+ /// A connect that fails mid-build releases the partial runtime (no orphaned client per
+ /// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing.
+ ///
+ [Fact]
+ public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds()
+ {
+ var attempts = 0;
+ await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
+ platformSource: new NoPlatforms())
+ {
+ SessionOpenOverrideForTests = _ => ++attempts == 1
+ ? throw new InvalidOperationException("gateway refused")
+ : Task.CompletedTask,
+ };
+
+ var ex = await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None));
+ ex.Message.ShouldBe("gateway refused");
+ driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client");
+ driver.OwnedMxSessionForTests.ShouldBeNull();
+ driver.GetHealth().State.ShouldBe(DriverState.Faulted);
+ driver.GetHealth().LastError.ShouldBe("gateway refused");
+
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ attempts.ShouldBe(2);
+ driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue();
+ driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+ }
+
+ /// DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op.
+ [Fact]
+ public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless()
+ {
+ var sub = new StreamingSubscriber();
+ var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
+ await driver.InitializeAsync("{}", CancellationToken.None);
+ await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
+
+ await driver.ShutdownAsync(CancellationToken.None);
+ await Should.NotThrowAsync(async () => await driver.DisposeAsync());
+ await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None));
+ await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None));
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs
new file mode 100644
index 00000000..aa6219c3
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs
@@ -0,0 +1,90 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// bounds the gateway CloseSession round-trip by
+/// the caller's token (the SDK dispose closes under its 30 s call timeout) and leaves the session
+/// re-openable.
+///
+public sealed class GalaxyMxSessionCloseTests
+{
+ /// A close against a never-answering gateway returns when the token fires, without throwing.
+ [Fact]
+ public async Task CloseAsync_returns_at_the_token_deadline_against_a_never_answering_gateway()
+ {
+ var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
+ {
+ OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
+ CloseOverrideForTests = ct => Task.Delay(Timeout.Infinite, ct),
+ };
+ await session.ConnectAsync(null!, CancellationToken.None);
+ session.IsConnected.ShouldBeTrue();
+
+ using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
+ var sw = System.Diagnostics.Stopwatch.StartNew();
+ await session.CloseAsync(cts.Token).WaitAsync(TimeSpan.FromSeconds(10));
+ sw.Stop();
+
+ sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5));
+ session.IsConnected.ShouldBeFalse();
+
+ // Still usable: a close is not a dispose.
+ await session.ConnectAsync(null!, CancellationToken.None);
+ session.IsConnected.ShouldBeTrue();
+ }
+
+ /// Positive control: a close that answers completes and reports the token it was handed.
+ [Fact]
+ public async Task CloseAsync_completes_when_the_gateway_answers()
+ {
+ CancellationToken seen = default;
+ var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
+ {
+ OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
+ CloseOverrideForTests = ct => { seen = ct; return Task.CompletedTask; },
+ };
+ await session.ConnectAsync(null!, CancellationToken.None);
+
+ using var cts = new CancellationTokenSource();
+ await session.CloseAsync(cts.Token);
+
+ seen.ShouldBe(cts.Token, "the caller's token must reach the gateway round-trip");
+ session.IsConnected.ShouldBeFalse();
+ }
+
+ ///
+ /// The driver's teardown is CloseAsync (bounded) then DisposeAsync (final). The dispose must
+ /// not issue a second gateway close; a fresh connect after a close must close again (positive
+ /// control: the guard keys on "was connected", not on "ever closed").
+ ///
+ [Fact]
+ public async Task CloseAsync_then_DisposeAsync_closes_the_gateway_session_once()
+ {
+ var closes = 0;
+ var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
+ {
+ OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
+ CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
+ };
+ await session.ConnectAsync(null!, CancellationToken.None);
+
+ await session.CloseAsync(CancellationToken.None);
+ await session.DisposeAsync();
+ closes.ShouldBe(1, "dispose after a close must not close the gateway session again");
+
+ var reopened = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
+ {
+ OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
+ CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
+ };
+ await reopened.ConnectAsync(null!, CancellationToken.None);
+ await reopened.CloseAsync(CancellationToken.None);
+ await reopened.ConnectAsync(null!, CancellationToken.None);
+ await reopened.DisposeAsync();
+ closes.ShouldBe(3, "each live session closes exactly once");
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs
index bfef6df2..6b311f9f 100644
--- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs
@@ -14,6 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// rather than asserting on internal state, so the
/// tests double as documentation of the listener-side contract.
///
+[Collection(ProcessWideTelemetryCollection.Name)]
public sealed class GalaxyTelemetryTests
{
/// Subscribes an ActivityListener for the test, captures each spawned activity.
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs
new file mode 100644
index 00000000..51b4cdfb
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs
@@ -0,0 +1,170 @@
+using System.Collections.Concurrent;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// Review 2026-10-21 G-8: a batch of writes ran strictly serially, three round-trips per
+/// first-touch tag. Distinct tags now run as concurrent lanes bounded at
+/// ; same-tag writes keep their
+/// order inside one lane; results stay at their request's index.
+///
+public sealed class GatewayGalaxyDataWriterConcurrencyTests
+{
+ private const int WaitMs = 10_000;
+
+ private static GatewayGalaxyDataWriter NewWriter()
+ => new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0);
+
+ ///
+ /// Fake ops whose AddItem parks every caller until callers are in
+ /// flight at once (or a safety delay elapses), recording the peak concurrency.
+ ///
+ private sealed class GatingOps : IMxWriteOps
+ {
+ private int _inFlight;
+ private int _peak;
+ private int _nextHandle;
+ private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly ConcurrentDictionary _handles = new(StringComparer.OrdinalIgnoreCase);
+
+ public int ReleaseAt { get; init; } = int.MaxValue;
+ public int Peak => Volatile.Read(ref _peak);
+ public ConcurrentQueue<(int Handle, MxValue Value)> RawWrites { get; } = new();
+ public ConcurrentQueue AddItems { get; } = new();
+ public Func? WriteResponder { get; init; }
+
+ /// Deterministic handle per tag when set; otherwise handles are assigned in AddItem completion order.
+ public Func? HandleFor { get; init; }
+
+ public string SessionId => "sess";
+ public int ServerHandle => 1;
+
+ public async Task AddItemAsync(string fullRef, CancellationToken ct)
+ {
+ AddItems.Enqueue(fullRef);
+ var now = Interlocked.Increment(ref _inFlight);
+ int seen;
+ do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen);
+ if (now >= ReleaseAt) _release.TrySetResult();
+ // Park on the signal; the delay is only a safety net against a wedged test (a starved
+ // pool must not release the gate early and turn the peak assertion flaky).
+ await Task.WhenAny(_release.Task, Task.Delay(WaitMs, ct));
+ Interlocked.Decrement(ref _inFlight);
+ return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle));
+ }
+
+ public Task InvokeAsync(MxCommandRequest request, CancellationToken ct)
+ => Task.FromResult(new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } });
+
+ public Task WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct)
+ {
+ RawWrites.Enqueue((itemHandle, value));
+ return Task.FromResult(WriteResponder?.Invoke(itemHandle) ?? GoodStatusRow());
+ }
+
+ // A reply carrying a Good MX status ROW (not an empty statuses array): the writer's static
+ // meters are process-wide, and an empty-statuses Good would increment galaxy.writes.unconfirmed
+ // under the meter tests running in parallel in another class.
+ private static MxCommandReply GoodStatusRow()
+ {
+ var reply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } };
+ reply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok });
+ return reply;
+ }
+ }
+
+ private static SecurityClassification Free(string _) => SecurityClassification.FreeAccess;
+
+ /// Sixteen distinct tags: lanes overlap (not serial) and never exceed the cap of eight.
+ [Fact]
+ public async Task Distinct_tags_write_concurrently_bounded_at_the_lane_cap()
+ {
+ var ops = new GatingOps { ReleaseAt = GatewayGalaxyDataWriter.MaxConcurrentWriteLanes };
+ var writes = Enumerable.Range(0, 16).Select(i => new WriteRequest($"Tag{i}.PV", (double)i)).ToArray();
+
+ var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
+ .WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+
+ ops.Peak.ShouldBe(GatewayGalaxyDataWriter.MaxConcurrentWriteLanes,
+ "first-touch lanes must overlap (serial = 1) and stay bounded (unbounded > 8)");
+ results.Count.ShouldBe(16);
+ results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
+ ops.AddItems.Count.ShouldBe(16, "one AddItem per distinct tag");
+ ops.RawWrites.Count.ShouldBe(16);
+ }
+
+ /// Writes to the same tag stay in input order inside their lane and share one AddItem.
+ [Fact]
+ public async Task Same_tag_writes_stay_ordered_and_share_one_handle()
+ {
+ var ops = new GatingOps { ReleaseAt = 2 };
+ var writes = new[]
+ {
+ new WriteRequest("Tank01.SP", 1.0),
+ new WriteRequest("Tank02.SP", 10.0),
+ new WriteRequest("Tank01.SP", 2.0),
+ new WriteRequest("Tank01.SP", 3.0),
+ };
+
+ var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
+ .WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+
+ results.Count.ShouldBe(4);
+ results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
+ ops.AddItems.Count(a => a == "Tank01.SP").ShouldBe(1, "a lane AddItems its tag once");
+ var tank01 = ops.RawWrites.Where(w => w.Handle == ops.RawWrites.First(x => x.Value.DoubleValue == 1.0).Handle)
+ .Select(w => w.Value.DoubleValue).ToArray();
+ tank01.ShouldBe([1.0, 2.0, 3.0], "same-tag writes must keep their authored order");
+ }
+
+ /// A per-tag failure lands at that request's index while the other lanes stay Good.
+ [Fact]
+ public async Task Results_land_at_their_request_index()
+ {
+ var ops = new GatingOps
+ {
+ ReleaseAt = 3,
+ HandleFor = tag => tag[0] - 'A' + 1, // A.PV → 1, B.PV → 2, C.PV → 3
+ WriteResponder = handle => new MxCommandReply
+ {
+ ProtocolStatus = new ProtocolStatus
+ {
+ Code = handle == 2 ? ProtocolStatusCode.MxaccessFailure : ProtocolStatusCode.Ok,
+ },
+ },
+ };
+ var writes = new[]
+ {
+ new WriteRequest("A.PV", 1.0),
+ new WriteRequest("B.PV", 2.0), // refused by the gateway
+ new WriteRequest("C.PV", 3.0),
+ };
+
+ var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
+ .WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+
+ results.Select(r => r.StatusCode).ShouldBe(
+ [StatusCodeMap.Good, StatusCodeMap.BadCommunicationError, StatusCodeMap.Good],
+ "each result must sit at its request's index regardless of lane completion order");
+ }
+
+ /// A single-tag batch takes the direct path and still produces one result per request.
+ [Fact]
+ public async Task Single_tag_batch_runs_serially_in_order()
+ {
+ var ops = new GatingOps { ReleaseAt = 1 };
+ var writes = new[] { new WriteRequest("X.PV", 1.0), new WriteRequest("X.PV", 2.0) };
+
+ var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
+ .WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
+
+ results.Count.ShouldBe(2);
+ ops.Peak.ShouldBe(1);
+ ops.RawWrites.Select(w => w.Value.DoubleValue).ShouldBe([1.0, 2.0]);
+ }
+}
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs
index 67a01d77..83d41dea 100644
--- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs
@@ -13,6 +13,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal
/// seam (the SDK session types are sealed + internal-ctor and cannot be faked).
///
+[Collection(ProcessWideTelemetryCollection.Name)]
public sealed class GatewayGalaxyDataWriterFailClosedTests
{
private static GatewayGalaxyDataWriter NewWriter()
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs
new file mode 100644
index 00000000..4513efa6
--- /dev/null
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs
@@ -0,0 +1,92 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+///
+/// Review 2026-10-21 G-3: allocated a
+/// new per recovery cycle and only the last one was ever
+/// disposed. Each cycle now releases its own CTS when its loop exits, and
+/// awaits the loop instead of blocking on it.
+///
+public sealed class ReconnectSupervisorCycleCtsTests
+{
+ private const int WaitMs = 10_000;
+
+ private static ReconnectOptions FastBackoff() =>
+ new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
+
+ /// Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control).
+ [Fact]
+ public async Task Every_completed_recovery_cycle_releases_its_own_cts()
+ {
+ var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var hold = new[] { false }; // read on the recovery loop's thread, written here — Volatile both ways
+ using var sup = new ReconnectSupervisor(
+ reopen: async _ =>
+ {
+ if (Volatile.Read(ref hold[0]))
+ {
+ reopenEntered.TrySetResult();
+ await holdReopen.Task;
+ }
+ },
+ replay: _ => Task.CompletedTask,
+ options: FastBackoff());
+
+ for (var i = 0; i < 3; i++)
+ {
+ sup.ReportTransportFailure(new IOException($"drop {i}"));
+ await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
+ }
+ await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS");
+ sup.CyclesStarted.ShouldBe(3);
+
+ // Positive control: a cycle parked inside reopen still holds its CTS.
+ Volatile.Write(ref hold[0], true);
+ sup.ReportTransportFailure(new IOException("drop 3"));
+ (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
+ sup.CyclesStarted.ShouldBe(4);
+ sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits");
+
+ holdReopen.TrySetResult();
+ await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
+ await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit");
+ }
+
+ /// StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking.
+ [Fact]
+ public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts()
+ {
+ var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var sup = new ReconnectSupervisor(
+ reopen: async ct =>
+ {
+ reopenEntered.TrySetResult();
+ await Task.Delay(Timeout.Infinite, ct);
+ },
+ replay: _ => Task.CompletedTask,
+ options: FastBackoff());
+
+ sup.ReportTransportFailure(new IOException("drop"));
+ (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
+
+ var stop = sup.StopAsync();
+ (await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return");
+ sup.CyclesReleased.ShouldBe(1);
+ await Should.NotThrowAsync(sup.StopAsync); // idempotent
+ Should.NotThrow(sup.Dispose);
+ }
+
+ private static async Task WaitUntilAsync(Func condition, string because)
+ {
+ var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
+ while (!condition())
+ {
+ if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
+ await Task.Delay(10);
+ }
+ }
+}