Merge lane F: Galaxy driver teardown, probing, knobs (#735)

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 13:50:53 -04:00
28 changed files with 3365 additions and 181 deletions
@@ -61,8 +61,12 @@ public sealed class GalaxyDiscoverer
/// </summary>
/// <param name="builder">The address space builder to populate with discovery results.</param>
/// <param name="cancellationToken">The cancellation token for the operation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken)
/// <returns>
/// The hierarchy this pass fetched, so a caller that needs the object set for something
/// else (the driver selects its <c>$WinPlatform</c> / <c>$AppEngine</c> probe targets from
/// it) reuses the one sweep instead of asking the gateway for the full galaxy twice.
/// </returns>
public async Task<IReadOnlyList<GalaxyObject>> DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(builder);
var objects = await _source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false);
@@ -132,6 +136,8 @@ public sealed class GalaxyDiscoverer
}
}
}
return objects;
}
// Resolve (and memoise) the folder for a gobject, recursively creating its parent's folder
@@ -5,22 +5,83 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
/// <summary>
/// Default <see cref="IGalaxyHierarchySource"/> wrapping the gateway's
/// <see cref="GalaxyRepositoryClient"/>. Pages internally via the client's overload.
/// <see cref="GalaxyRepositoryClient"/>. Pages the <c>DiscoverHierarchy</c> RPC itself,
/// at the configured <c>Repository.DiscoverPageSize</c>: the client's typed
/// <c>DiscoverHierarchyAsync</c> overloads hard-code a 5000-row page
/// (<c>DiscoverHierarchyOptions</c> carries no page size), so the knob the AdminUI
/// authors and the factory parses used to reach nothing. The raw RPC takes the page
/// size on the wire (<c>DiscoverHierarchyRequest.page_size</c>), so it is honoured here.
/// </summary>
public sealed class GatewayGalaxyHierarchySource : IGalaxyHierarchySource
{
/// <summary>The page size used when the configured one is not positive — the client library's own default.</summary>
public const int DefaultPageSize = 5000;
private readonly GalaxyRepositoryClient _client;
private readonly int _pageSize;
/// <summary>
/// Initializes a new instance of the <see cref="GatewayGalaxyHierarchySource"/> class.
/// </summary>
/// <param name="client">The gateway's Galaxy repository client for discovering the object hierarchy.</param>
public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client)
/// <param name="pageSize">
/// Objects per <c>DiscoverHierarchy</c> page (<c>Repository.DiscoverPageSize</c>). A value that is
/// not positive falls back to <see cref="DefaultPageSize"/>.
/// </param>
public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client, int pageSize = DefaultPageSize)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_pageSize = EffectivePageSize(pageSize);
}
/// <summary>The page size this source sends on every <c>DiscoverHierarchy</c> request.</summary>
public int PageSize => _pageSize;
/// <summary>Clamp a configured page size to something the gateway accepts: non-positive → <see cref="DefaultPageSize"/>.</summary>
/// <param name="configured">The configured value.</param>
/// <returns>The page size to send.</returns>
public static int EffectivePageSize(int configured) => configured > 0 ? configured : DefaultPageSize;
/// <inheritdoc />
public Task<IReadOnlyList<GalaxyObject>> GetHierarchyAsync(CancellationToken cancellationToken)
=> _client.DiscoverHierarchyAsync(cancellationToken);
=> PageAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
/// <summary>
/// Walk every <c>DiscoverHierarchy</c> page at <paramref name="pageSize"/> and concatenate the
/// objects. Mirrors the client library's own loop (including its repeated-page-token guard,
/// which turns a misbehaving gateway into an error rather than an endless walk), with the
/// RPC behind a delegate so the loop is unit-testable without a channel.
/// </summary>
/// <param name="discover">The raw <c>DiscoverHierarchy</c> RPC.</param>
/// <param name="pageSize">Objects per page; sent on every request.</param>
/// <param name="cancellationToken">Cancellation token for the walk.</param>
/// <returns>Every object across all pages, in gateway order.</returns>
internal static async Task<IReadOnlyList<GalaxyObject>> PageAsync(
Func<DiscoverHierarchyRequest, CancellationToken, Task<DiscoverHierarchyReply>> discover,
int pageSize,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(discover);
var effective = EffectivePageSize(pageSize);
var objects = new List<GalaxyObject>();
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
var pageToken = string.Empty;
do
{
var request = new DiscoverHierarchyRequest { PageSize = effective, PageToken = pageToken };
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
objects.AddRange(reply.Objects);
pageToken = reply.NextPageToken ?? string.Empty;
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
{
throw new InvalidOperationException(
$"Galaxy DiscoverHierarchy returned a repeated page token '{pageToken}' — refusing to loop.");
}
}
while (!string.IsNullOrWhiteSpace(pageToken));
return objects;
}
}
@@ -0,0 +1,88 @@
using ZB.MOM.WW.MxGateway.Client;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
/// <summary>
/// Default <see cref="IGalaxyPlatformSource"/>: a platform-only <c>DiscoverHierarchy</c> sweep
/// over the gateway's <see cref="GalaxyRepositoryClient"/> — the request filters on
/// <c>template_chain_contains</c> (<c>$WinPlatform</c>, <c>$AppEngine</c>) and asks for no
/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the
/// address-space discovery pulls. Paged at the configured <c>Repository.DiscoverPageSize</c>.
/// </summary>
public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
{
/// <summary>Template-chain tokens that identify a platform / engine object.</summary>
public static readonly IReadOnlyList<string> PlatformTemplates = ["$WinPlatform", "$AppEngine"];
private readonly GalaxyRepositoryClient _client;
private readonly int _pageSize;
/// <summary>Initializes a new instance of the <see cref="GatewayGalaxyPlatformSource"/> class.</summary>
/// <param name="client">The gateway's Galaxy repository client.</param>
/// <param name="pageSize">Objects per page (<c>Repository.DiscoverPageSize</c>); non-positive → the client default.</param>
public GatewayGalaxyPlatformSource(GalaxyRepositoryClient client, int pageSize = GatewayGalaxyHierarchySource.DefaultPageSize)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_pageSize = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize);
}
/// <inheritdoc />
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
=> QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
/// <summary>
/// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every
/// request carries the template filter and <c>include_attributes = false</c>; the reply is
/// re-filtered on the returned template chain (a gateway that ignores the filter still
/// yields only platforms) and reduced to non-empty tag names.
/// </summary>
/// <param name="discover">The raw <c>DiscoverHierarchy</c> RPC.</param>
/// <param name="pageSize">Objects per page.</param>
/// <param name="cancellationToken">Cancellation token for the walk.</param>
/// <returns>Distinct platform tag names in gateway order.</returns>
internal static async Task<IReadOnlyList<string>> QueryAsync(
Func<DiscoverHierarchyRequest, CancellationToken, Task<DiscoverHierarchyReply>> discover,
int pageSize,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(discover);
var effective = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize);
var names = new List<string>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
var pageToken = string.Empty;
do
{
var request = new DiscoverHierarchyRequest
{
PageSize = effective,
PageToken = pageToken,
IncludeAttributes = false,
};
request.TemplateChainContains.Add(PlatformTemplates);
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
foreach (var obj in reply.Objects)
{
if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
if (seen.Add(obj.TagName)) names.Add(obj.TagName);
}
pageToken = reply.NextPageToken ?? string.Empty;
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
{
throw new InvalidOperationException(
$"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop.");
}
}
while (!string.IsNullOrWhiteSpace(pageToken));
return names;
}
private static bool IsPlatform(GalaxyObject obj) =>
obj.TemplateChain.Count == 0 // a gateway that strips the chain already filtered server-side
|| obj.TemplateChain.Any(t => PlatformTemplates.Contains(t, StringComparer.OrdinalIgnoreCase));
}
@@ -0,0 +1,18 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
/// <summary>
/// Driver-side seam for the set of Galaxy platform / engine objects whose <c>ScanState</c>
/// the <c>PerPlatformProbeWatcher</c> advises. Production wraps the gateway's
/// <c>DiscoverHierarchy</c> RPC filtered to <c>$WinPlatform</c> / <c>$AppEngine</c> with no
/// attributes (<see cref="GatewayGalaxyPlatformSource"/>); tests substitute a canned list.
/// The driver seeds the watcher from this at <c>InitializeAsync</c> and re-seeds on every
/// deploy-watcher rediscovery — the runtime never calls <c>DiscoverAsync</c> on a live driver,
/// so without this seed the probe watcher never had a platform set in production.
/// </summary>
public interface IGalaxyPlatformSource
{
/// <summary>The <c>tag_name</c> of every deployed <c>$WinPlatform</c> / <c>$AppEngine</c> object.</summary>
/// <param name="cancellationToken">Cancellation token for the query.</param>
/// <returns>Platform tag names; empty when the galaxy has none.</returns>
Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken);
}
File diff suppressed because it is too large Load Diff
@@ -54,12 +54,18 @@ public sealed class GalaxyDriverProbe : IDriverProbe
if (string.IsNullOrWhiteSpace(host) || port <= 0)
return new(false, "Config has no host/port to probe.", null);
// The config's own ProbeTimeoutSeconds caps the probe deadline. It can only tighten the
// caller's budget, never extend it — the caller's token fires at `timeout` regardless.
timeout = EffectiveTimeout(timeout, opts.ProbeTimeoutSeconds);
using var probeCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
probeCts.CancelAfter(timeout);
// --- TCP preflight: fast-fail for closed ports / unreachable hosts ---
var sw = Stopwatch.StartNew();
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(host, port, ct);
await socket.ConnectAsync(host, port, probeCts.Token);
}
catch (SocketException ex)
{
@@ -86,10 +92,7 @@ public sealed class GalaxyDriverProbe : IDriverProbe
{
client = GalaxyRepositoryClient.Create(BuildProbeClientOptions(opts.Gateway, timeout));
using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
deadlineCts.CancelAfter(timeout);
await client.TestConnectionAsync(deadlineCts.Token).ConfigureAwait(false);
await client.TestConnectionAsync(probeCts.Token).ConfigureAwait(false);
sw.Stop();
// No RpcException ⇒ the gateway answered the gRPC call successfully.
@@ -157,6 +160,26 @@ public sealed class GalaxyDriverProbe : IDriverProbe
_ => (false, $"Reachable at {host}:{port} but gateway gRPC handshake failed: {code}"),
};
/// <summary>
/// The deadline the probe actually runs under: the caller's budget, tightened by the
/// config's <see cref="GalaxyDriverOptions.ProbeTimeoutSeconds"/> when that is positive and
/// shorter. The knob was authored by the AdminUI and parsed by the factory but consumed by
/// nothing — the probe always ran on the caller's budget alone. A non-positive caller budget
/// is floored to 1 s before the comparison (the same floor <see cref="BuildProbeClientOptions"/>
/// applies to the gRPC call budget), so the probe runs briefly rather than cancelling before
/// it starts; a longer config value never raises it above that floor.
/// </summary>
/// <param name="callerTimeout">The budget the host handed the probe.</param>
/// <param name="configuredSeconds">The config's <c>ProbeTimeoutSeconds</c>; ignored when not positive.</param>
/// <returns>The effective probe deadline.</returns>
internal static TimeSpan EffectiveTimeout(TimeSpan callerTimeout, int configuredSeconds)
{
var caller = callerTimeout > TimeSpan.Zero ? callerTimeout : TimeSpan.FromSeconds(1);
if (configuredSeconds <= 0) return caller;
var configured = TimeSpan.FromSeconds(configuredSeconds);
return configured < caller ? configured : caller;
}
/// <summary>
/// Builds the gRPC client options for the probe ping from the gateway config WITHOUT
/// resolving the API-key secret ref (the host owns secret resolution; the probe sends the
@@ -38,12 +38,19 @@ public sealed class PerPlatformProbeWatcher : IDisposable
private readonly ILogger _logger;
private readonly int _bufferedUpdateIntervalMs;
// The driver's fan-out registry. The EventPump resolves every inbound event through it, so
// the probe handles must be registered there (under SubscriptionRegistry.ProbeSubscriptionId)
// or every ScanState event is dropped as "stale after unsubscribe". Null when the watcher is
// driven directly (unit tests of the decoding table).
private readonly SubscriptionRegistry? _registry;
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
private readonly ConcurrentDictionary<string, int> _itemHandlesByPlatform =
new(StringComparer.OrdinalIgnoreCase);
private readonly Lock _syncLock = new();
private bool _disposed;
// Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop.
private volatile bool _disposed;
/// <summary>Initializes a new instance of the PerPlatformProbeWatcher class.</summary>
/// <param name="subscriber">The Galaxy subscriber for managing probe subscriptions.</param>
@@ -55,9 +62,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable
HostStatusAggregator aggregator,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
: this(subscriber, aggregator, registry: null, logger, bufferedUpdateIntervalMs)
{
}
/// <summary>
/// Driver-side ctor: the watcher publishes its probe handles into <paramref name="registry"/>
/// under <see cref="SubscriptionRegistry.ProbeSubscriptionId"/> so the <see cref="EventPump"/>
/// fans ScanState events out to <see cref="OnProbeValueChanged"/>.
/// </summary>
/// <param name="subscriber">The Galaxy subscriber for managing probe subscriptions.</param>
/// <param name="aggregator">The host status aggregator for tracking platform connectivity.</param>
/// <param name="registry">The driver's subscription registry; null to keep the handles private.</param>
/// <param name="logger">Optional logger for diagnostic messages.</param>
/// <param name="bufferedUpdateIntervalMs">Buffered update interval in milliseconds; must be >= 0.</param>
internal PerPlatformProbeWatcher(
IGalaxySubscriber subscriber,
HostStatusAggregator aggregator,
SubscriptionRegistry? registry,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
_registry = registry;
_logger = logger ?? NullLogger.Instance;
if (bufferedUpdateIntervalMs < 0)
{
@@ -67,6 +95,103 @@ public sealed class PerPlatformProbeWatcher : IDisposable
_bufferedUpdateIntervalMs = bufferedUpdateIntervalMs;
}
/// <summary>Current platform → probe-handle bindings, as the registry sees them. Test seam.</summary>
internal IReadOnlyList<TagBinding> CurrentBindings => BuildBindings();
private IReadOnlyList<TagBinding> BuildBindings() =>
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))];
/// <summary>
/// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped —
/// a stopped watcher must never write its entry back into a registry the teardown cleared).
/// </summary>
private void PublishBindings()
{
lock (_syncLock) PublishBindingsLocked();
}
private void PublishBindingsLocked()
{
if (_registry is null || _disposed) return;
var bindings = BuildBindings();
if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId);
else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings);
}
/// <summary>
/// Re-issue SubscribeBulk for every tracked platform and replace the handles — the
/// reconnect replay path, where the gateway session was recreated and every pre-reconnect
/// handle is dead. Platforms the gateway now rejects keep a 0 handle (no events expected).
/// </summary>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>The number of probe references re-subscribed.</returns>
public async Task<int> ResubscribeAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
var platforms = _itemHandlesByPlatform.Keys.ToArray();
if (platforms.Length == 0) return 0;
await SubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false);
return platforms.Length;
}
private async Task SubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
{
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
// changes are rare so a coarser interval is usually fine; deployments that need
// tighter health visibility can dial it down through GalaxyDriverOptions.
var results = await _subscriber.SubscribeBulkAsync(
probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
var acquired = new (string Platform, int ItemHandle, string? Error)[platforms.Count];
for (var i = 0; i < platforms.Count; i++)
{
var match = results.FirstOrDefault(r => string.Equals(
r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
acquired[i] = (platforms[i], match is { WasSuccessful: true } ? match.ItemHandle : 0,
match?.ErrorMessage ?? "<no result returned>");
}
// Re-check the stop AFTER the await, atomically with the record + publish: a StopAsync that
// ran while SubscribeBulk was in flight already cleared the map and the registry entry, so
// recording now would resurrect the probe entry into the next runtime's registry.
bool stopped;
lock (_syncLock)
{
stopped = _disposed;
if (!stopped)
{
foreach (var (platform, itemHandle, _) in acquired) _itemHandlesByPlatform[platform] = itemHandle;
PublishBindingsLocked();
}
}
if (stopped)
{
var orphans = acquired.Where(a => a.ItemHandle > 0).Select(a => a.ItemHandle).ToArray();
if (orphans.Length > 0)
{
try { await _subscriber.UnsubscribeBulkAsync(orphans, CancellationToken.None).ConfigureAwait(false); }
catch (Exception ex)
{
_logger.LogWarning(ex,
"PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length);
}
}
return;
}
foreach (var (platform, itemHandle, error) in acquired)
{
if (itemHandle <= 0)
{
_logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error);
}
}
}
/// <summary>Snapshot of platform tag names currently watched.</summary>
public IReadOnlyCollection<string> WatchedPlatforms => [.. _itemHandlesByPlatform.Keys];
@@ -109,6 +234,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable
if (toRemove.Count > 0)
{
// Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe.
PublishBindings();
try
{
await _subscriber.UnsubscribeBulkAsync(
@@ -124,30 +251,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable
if (toAdd.Count == 0) return;
var probeAddresses = toAdd.Select(p => p + ProbeSuffix).ToArray();
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
// changes are rare so a coarser interval is usually fine; deployments that need
// tighter health visibility can dial it down through GalaxyDriverOptions.
var results = await _subscriber.SubscribeBulkAsync(
probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
for (var i = 0; i < toAdd.Count; i++)
{
var platform = toAdd[i];
var match = results.FirstOrDefault(r => string.Equals(
r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0;
_itemHandlesByPlatform[platform] = itemHandle;
if (itemHandle <= 0)
{
_logger.LogWarning(
"PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}",
platform, match?.ErrorMessage ?? "<no result returned>");
}
}
await SubscribeAndRecordAsync(toAdd, cancellationToken).ConfigureAwait(false);
}
/// <summary>
@@ -196,20 +300,32 @@ public sealed class PerPlatformProbeWatcher : IDisposable
};
}
/// <summary>Disposes the probe watcher and unsubscribes all tracked platforms.</summary>
public void Dispose()
/// <summary>
/// Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway
/// (best-effort — a transport error is logged and swallowed). The async counterpart of
/// <see cref="Dispose"/>; the driver's <c>ShutdownAsync</c> uses it so the unsubscribe
/// round-trip is awaited rather than blocked on. Idempotent.
/// </summary>
/// <param name="cancellationToken">Bounds the unsubscribe round-trip.</param>
/// <returns>A task that completes once the handles have been released (or the attempt gave up).</returns>
public async Task StopAsync(CancellationToken cancellationToken)
{
if (_disposed) return;
_disposed = true;
// Best-effort unsubscribe everything we know about. Run synchronously through
// GetAwaiter().GetResult() since Dispose is sync; transport errors are swallowed.
var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray();
_itemHandlesByPlatform.Clear();
int[] liveHandles;
lock (_syncLock)
{
if (_disposed) return;
_disposed = true;
liveHandles = [.. _itemHandlesByPlatform.Values.Where(h => h > 0)];
_itemHandlesByPlatform.Clear();
_registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId);
}
if (liveHandles.Length > 0)
{
try { _subscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None).GetAwaiter().GetResult(); }
catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher dispose unsubscribe failed"); }
try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); }
catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); }
}
}
/// <summary>Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip).</summary>
public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult();
}
@@ -187,9 +187,11 @@ internal sealed class EventPump : IAsyncDisposable
NotifyStreamFault(cause);
}
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
catch (Exception) when (ct.IsCancellationRequested)
{
// Clean shutdown — no log.
// Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled)
// rather than OperationCanceledException; either way the pump was told to stop, so this
// is not a transport fault and must not wake the reconnect supervisor mid-teardown.
}
catch (Exception ex)
{
@@ -283,14 +285,19 @@ internal sealed class EventPump : IAsyncDisposable
EventsDispatched.Add(1, _clientTag);
}
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
catch (Exception) when (ct.IsCancellationRequested)
{
// Clean shutdown.
}
catch (Exception ex)
{
// The read loop is still draining the stream into a channel nobody reads: the pump is
// as dead as after a transport fault, but used to raise nothing — health stayed
// Healthy and the comms-loss watchdog never tripped (a clean termination evading the
// watchdog). Report it as a stream fault so the supervisor recreates the pump.
_logger.LogWarning(ex,
"Galaxy EventPump dispatch loop ended with an exception — events past this point will be lost until restart.");
"Galaxy EventPump dispatch loop ended with an exception — notifying reconnect supervisor so the pump is recreated.");
NotifyStreamFault(ex);
}
}
@@ -60,6 +60,12 @@ public sealed class GalaxyMxSession : IAsyncDisposable
/// </summary>
internal Func<CancellationToken, Task>? OpenAndRegisterOverrideForTests { get; set; }
/// <summary>
/// Test seam — replaces the gateway <c>CloseSession</c> round-trip in <see cref="CloseAsync"/>
/// so the bounded-close behaviour can be exercised against a never-answering fake.
/// </summary>
internal Func<CancellationToken, Task>? CloseOverrideForTests { get; set; }
/// <summary>
/// Connect the underlying gateway client + open an MXAccess session + register the
/// configured client name. Idempotent — second calls are no-ops while a connect has
@@ -96,7 +102,7 @@ public sealed class GalaxyMxSession : IAsyncDisposable
{
// A partial open (e.g. OpenSession succeeded but Register threw) must not leave a half-open
// handle that blocks the next reconnect. Tear the partial state down so a retry rebuilds cleanly.
await TeardownSessionAsync().ConfigureAwait(false);
await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false);
throw;
}
}
@@ -112,10 +118,23 @@ public sealed class GalaxyMxSession : IAsyncDisposable
public async Task RecreateAsync(MxGatewayClientOptions clientOptions, CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
await TeardownSessionAsync().ConfigureAwait(false);
await TeardownSessionAsync(cancellationToken).ConfigureAwait(false);
await ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Close the gateway session and release the owned client under
/// <paramref name="cancellationToken"/>. The SDK's <c>DisposeAsync</c> closes with NO token,
/// i.e. under the 30 s <c>DefaultCallTimeout</c> — against a blackholed gateway that parks
/// the driver's <c>ShutdownAsync</c> (and the host's <c>PostStop</c>) for the full timeout.
/// A close that is cancelled or fails is NOT retried through the SDK dispose (which would
/// park again); the channel is dropped and the gateway reaps the session on its idle lease.
/// The session can be re-opened afterwards (this does not set <c>_disposed</c>).
/// </summary>
/// <param name="cancellationToken">Bounds the gateway <c>CloseSession</c> round-trip.</param>
/// <returns>A task that completes once the session and client are released.</returns>
public Task CloseAsync(CancellationToken cancellationToken) => TeardownSessionAsync(cancellationToken);
/// <summary>
/// Test seam — attach a session opened externally (e.g. against an in-process gw
/// fake). Skips the gateway-client construction so tests can drive the session
@@ -143,24 +162,45 @@ public sealed class GalaxyMxSession : IAsyncDisposable
{
if (_disposed) return;
_disposed = true;
await TeardownSessionAsync().ConfigureAwait(false);
await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false);
}
/// <summary>
/// Resets the connect flag and best-effort disposes + nulls the owned session and client.
/// Shared by <see cref="DisposeAsync"/> (final teardown) and <see cref="RecreateAsync"/> /
/// the partial-open recovery path (rebuild teardown) — it does NOT set <c>_disposed</c>, so
/// a torn-down session can be re-opened.
/// Resets the connect flag and best-effort closes + nulls the owned session and client.
/// Shared by <see cref="DisposeAsync"/> (final teardown), <see cref="CloseAsync"/> (bounded
/// teardown) and <see cref="RecreateAsync"/> / the partial-open recovery path (rebuild
/// teardown) — it does NOT set <c>_disposed</c>, so a torn-down session can be re-opened.
/// </summary>
private async Task TeardownSessionAsync()
private async Task TeardownSessionAsync(CancellationToken cancellationToken)
{
var wasConnected = _connected;
_connected = false;
// The override models the gateway round-trip, so it runs only when there was a live
// session to close — a second teardown (CloseAsync then DisposeAsync) must not close twice.
var closeOverride = CloseOverrideForTests;
if (closeOverride is not null && wasConnected)
{
try { await closeOverride(cancellationToken).ConfigureAwait(false); }
catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession close failed or timed out (best-effort)"); }
}
if (_session is not null)
{
try { await _session.DisposeAsync().ConfigureAwait(false); }
catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession session dispose failed (best-effort)"); }
var session = _session;
_session = null;
try
{
// CloseAsync(ct) is the bounded gateway round-trip; DisposeAsync only re-runs an
// un-replied close (with no token), so it is reached only after a successful one.
await session.CloseAsync(cancellationToken).ConfigureAwait(false);
await session.DisposeAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"GalaxyMxSession close failed or timed out (best-effort) — dropping the channel; the gateway reaps the session on its idle lease.");
}
}
if (_ownedClient is not null)
@@ -120,15 +120,87 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
// Bind the per-batch MX write operations behind the internal IMxWriteOps seam (see below) so the
// whole write pipeline is unit-testable without the sealed SDK session (archreview 06/S-1).
var ops = new SessionMxWriteOps(session, serverHandle);
return await WriteBatchAsync(ops, writes, securityResolver, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Upper bound on write lanes in flight at once. A first-touch write is three round-trips
/// (AddItem → AdviseSupervisory → Write) and the batch used to run them strictly serially;
/// distinct tags are independent, so their lanes now overlap, bounded so a large batch cannot
/// flood the gateway's STA pump. Acks stay serial by design (see <c>GalaxyDriver.AcknowledgeAsync</c>).
/// </summary>
internal const int MaxConcurrentWriteLanes = 8;
/// <summary>Test seam: drive the real batch pipeline against a fake <see cref="IMxWriteOps"/>.</summary>
/// <param name="ops">The (fake) MX write operations.</param>
/// <param name="writes">The batch.</param>
/// <param name="securityResolver">Per-tag classification resolver.</param>
/// <param name="ct">The cancellation token.</param>
/// <returns>One result per request, in input order.</returns>
internal Task<IReadOnlyList<WriteResult>> WriteBatchForTestAsync(
IMxWriteOps ops, IReadOnlyList<WriteRequest> writes,
Func<string, SecurityClassification> securityResolver, CancellationToken ct)
=> WriteBatchAsync(ops, writes, securityResolver, ct);
/// <summary>
/// Run a batch as one lane per distinct tag (OrdinalIgnoreCase): writes to the SAME tag
/// stay in input order inside their lane, lanes run concurrently up to
/// <see cref="MaxConcurrentWriteLanes"/>, and results land at their request's index.
/// A single-tag batch takes the direct path with no gate.
/// </summary>
private async Task<IReadOnlyList<WriteResult>> WriteBatchAsync(
IMxWriteOps ops, IReadOnlyList<WriteRequest> writes,
Func<string, SecurityClassification> securityResolver, CancellationToken ct)
{
var results = new WriteResult[writes.Count];
if (writes.Count == 0) return results;
// Group request indices by tag, first-seen order preserved so the lane start order is
// deterministic and a same-tag sequence keeps its authored order.
var lanes = new Dictionary<string, List<int>>(StringComparer.OrdinalIgnoreCase);
var laneOrder = new List<List<int>>();
for (var i = 0; i < writes.Count; i++)
{
results[i] = await WriteOneAsync(ops, writes[i],
securityResolver(writes[i].FullReference), cancellationToken)
.ConfigureAwait(false);
if (!lanes.TryGetValue(writes[i].FullReference, out var lane))
{
lane = [];
lanes[writes[i].FullReference] = lane;
laneOrder.Add(lane);
}
lane.Add(i);
}
if (laneOrder.Count == 1)
{
await RunLaneAsync(laneOrder[0]).ConfigureAwait(false);
return results;
}
using var gate = new SemaphoreSlim(MaxConcurrentWriteLanes, MaxConcurrentWriteLanes);
var tasks = new Task[laneOrder.Count];
for (var l = 0; l < laneOrder.Count; l++)
{
var lane = laneOrder[l];
tasks[l] = RunGatedAsync(lane);
}
await Task.WhenAll(tasks).ConfigureAwait(false);
return results;
async Task RunGatedAsync(List<int> lane)
{
await gate.WaitAsync(ct).ConfigureAwait(false);
try { await RunLaneAsync(lane).ConfigureAwait(false); }
finally { gate.Release(); }
}
async Task RunLaneAsync(List<int> lane)
{
foreach (var i in lane)
{
results[i] = await WriteOneAsync(ops, writes[i],
securityResolver(writes[i].FullReference), ct).ConfigureAwait(false);
}
}
}
/// <summary>Test seam: drive the real single-write pipeline against a fake <see cref="IMxWriteOps"/>
@@ -62,6 +62,16 @@ public sealed class ReconnectSupervisor : IDisposable
private Task? _recoveryLoop;
private CancellationTokenSource? _loopCts;
private bool _disposed;
private int _cyclesStarted;
private int _cyclesReleased;
/// <summary>Recovery cycles started so far (one per <see cref="ReportTransportFailure"/> that left Healthy). Test seam.</summary>
internal int CyclesStarted => Volatile.Read(ref _cyclesStarted);
/// <summary>Recovery cycles whose <see cref="CancellationTokenSource"/> has been disposed. Equals
/// <see cref="CyclesStarted"/> once every loop has exited — a cycle's CTS used to be released only
/// if it happened to be the LAST one when <see cref="Dispose"/> ran. Test seam.</summary>
internal int CyclesReleased => Volatile.Read(ref _cyclesReleased);
/// <summary>Fires after every state transition.</summary>
public event EventHandler<StateTransition>? StateChanged;
@@ -136,8 +146,13 @@ public sealed class ReconnectSupervisor : IDisposable
TransitionLocked(State.TransportLost, cause.Message);
_loopCts = new CancellationTokenSource();
_recoveryLoop = Task.Run(() => RecoveryLoopAsync(_loopCts.Token));
// Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's
// finally). Before, every cycle allocated a fresh CTS and only the last one was ever
// disposed — one leaked CancellationTokenSource per outage for the driver's lifetime.
var cts = new CancellationTokenSource();
_loopCts = cts;
Interlocked.Increment(ref _cyclesStarted);
_recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts));
}
}
@@ -156,7 +171,26 @@ public sealed class ReconnectSupervisor : IDisposable
}
}
private async Task RecoveryLoopAsync(CancellationToken ct)
private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts)
{
try
{
await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false);
}
finally
{
// Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may
// race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object.
lock (_stateLock)
{
if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null;
}
cycleCts.Dispose();
Interlocked.Increment(ref _cyclesReleased);
}
}
private async Task RecoveryLoopCoreAsync(CancellationToken ct)
{
var attempt = 0;
while (!ct.IsCancellationRequested)
@@ -240,8 +274,13 @@ public sealed class ReconnectSupervisor : IDisposable
}
}
/// <summary>Disposes the supervisor and cancels any in-flight recovery loop.</summary>
public void Dispose()
/// <summary>
/// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's
/// thread. The async counterpart of <see cref="Dispose"/>; the driver's <c>ShutdownAsync</c>
/// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent.
/// </summary>
/// <returns>A task that completes once the recovery loop has exited.</returns>
public async Task StopAsync()
{
if (_disposed) return;
_disposed = true;
@@ -250,13 +289,18 @@ public sealed class ReconnectSupervisor : IDisposable
Task? loop;
lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }
cts?.Cancel();
// The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
// throws on Cancel, and a disposed one is a no-op on Dispose.
try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ }
if (loop is not null)
{
try { loop.GetAwaiter().GetResult(); } catch { /* shutdown */ }
try { await loop.ConfigureAwait(false); } catch { /* shutdown */ }
}
cts?.Dispose();
}
/// <summary>Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits).</summary>
public void Dispose() => StopAsync().GetAwaiter().GetResult();
}
/// <summary>
@@ -18,10 +18,39 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
/// </remarks>
internal sealed class SubscriptionRegistry
{
/// <summary>
/// Reserved subscription id for the per-platform <c>ScanState</c> probe handles
/// (<c>PerPlatformProbeWatcher</c>). Driver subscription ids are positive and
/// monotonic (<see cref="NextSubscriptionId"/>), so a negative id can never collide.
/// The probe handles used to live only in the watcher's own map, so the
/// <see cref="EventPump"/> — which resolves subscribers through THIS registry —
/// dropped every probe event as "stale after unsubscribe" and per-platform host
/// status never updated.
/// </summary>
public const long ProbeSubscriptionId = -1;
private readonly ConcurrentDictionary<long, SubscriptionEntry> _bySubscriptionId = new();
private readonly ConcurrentDictionary<int, ImmutableHashSet<long>> _subscribersByItemHandle = new();
private long _nextSubscriptionId;
/// <summary>True when <paramref name="subscriptionId"/> is currently registered.</summary>
/// <param name="subscriptionId">The subscription identifier.</param>
/// <returns>Whether the id is tracked.</returns>
public bool Contains(long subscriptionId) => _bySubscriptionId.ContainsKey(subscriptionId);
/// <summary>
/// <see cref="Rebind"/> when the id is already tracked (so its stale reverse-map entries
/// are dropped), <see cref="Register"/> otherwise. Used by the probe watcher, whose single
/// reserved subscription is re-issued on every platform-set change.
/// </summary>
/// <param name="subscriptionId">The subscription identifier.</param>
/// <param name="bindings">The tag bindings for the subscription.</param>
public void Upsert(long subscriptionId, IReadOnlyList<TagBinding> bindings)
{
if (Contains(subscriptionId)) Rebind(subscriptionId, bindings);
else Register(subscriptionId, bindings);
}
/// <summary>Gets the number of tracked subscriptions.</summary>
public int TrackedSubscriptionCount => _bySubscriptionId.Count;
/// <summary>Gets the number of tracked item handles.</summary>
@@ -100,6 +129,17 @@ internal sealed class SubscriptionRegistry
}
/// <summary>
/// Drop every subscription and the whole reverse map. Used by the driver's runtime teardown:
/// once the gateway session is closed every item handle in here is dead, and the host
/// re-issues its subscriptions (fresh ids) after the next <c>InitializeAsync</c>.
/// </summary>
public void Clear()
{
_bySubscriptionId.Clear();
_subscribersByItemHandle.Clear();
}
/// <summary>Snapshot every active binding for diagnostic output.</summary>
/// <returns>A flattened list of every tag binding across all tracked subscriptions.</returns>
public IReadOnlyList<TagBinding> SnapshotAllBindings() =>
@@ -92,7 +92,7 @@
<div class="col-md-3">
<label class="form-label">Discover page size</label>
<InputNumber @bind-Value="_form.Galaxy.RepositoryDiscoverPageSize" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<div class="form-text">Default 5000 objects per page.</div>
<div class="form-text">Objects per DiscoverHierarchy page the driver asks the gateway for. Default 5000.</div>
</div>
<div class="col-md-3">
<div class="form-check form-switch mt-4">
@@ -138,7 +138,7 @@
<div class="col-md-3">
<label class="form-label">Admin UI probe timeout (seconds)</label>
<InputNumber @bind-Value="_form.Galaxy.ProbeTimeoutSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<div class="form-text">Max 60. Used by Test Connect. Default 30.</div>
<div class="form-text">Caps the Test Connect probe deadline (the probe's own budget still applies, so this only shortens it). Max 60. Default 30.</div>
</div>
</div>
</div>
@@ -0,0 +1,92 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
/// <summary>
/// Review 2026-10-21 G-5: <c>Repository.DiscoverPageSize</c> was authored and parsed but reached
/// nothing — the client's typed <c>DiscoverHierarchyAsync</c> hard-codes 5000. The source now pages
/// the raw RPC itself at the configured size.
/// </summary>
public sealed class GatewayGalaxyHierarchySourcePagingTests
{
private static GalaxyObject Obj(int id) => new() { GobjectId = id, TagName = $"Obj{id}" };
/// <summary>Every request carries the configured page size and the page tokens chain until the last page.</summary>
[Fact]
public async Task Pages_at_the_configured_size_and_chains_tokens()
{
var requests = new List<DiscoverHierarchyRequest>();
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
{
requests.Add(req);
var reply = new DiscoverHierarchyReply();
switch (req.PageToken)
{
case "":
reply.Objects.Add([Obj(1), Obj(2)]);
reply.NextPageToken = "p2";
break;
case "p2":
reply.Objects.Add([Obj(3), Obj(4)]);
reply.NextPageToken = "p3";
break;
default:
reply.Objects.Add(Obj(5));
reply.NextPageToken = "";
break;
}
return Task.FromResult(reply);
}
var objects = await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 2, CancellationToken.None);
objects.Select(o => o.GobjectId).ShouldBe([1, 2, 3, 4, 5]);
requests.Count.ShouldBe(3);
requests.ShouldAllBe(r => r.PageSize == 2);
requests.Select(r => r.PageToken).ShouldBe(["", "p2", "p3"]);
}
/// <summary>A different configured size is what goes on the wire (positive control for the knob).</summary>
[Fact]
public async Task A_different_page_size_is_sent_on_the_wire()
{
int? sent = null;
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
{
sent = req.PageSize;
return Task.FromResult(new DiscoverHierarchyReply());
}
await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 250, CancellationToken.None);
sent.ShouldBe(250);
}
/// <summary>A non-positive configured size falls back to the client library's 5000 default.</summary>
[Theory]
[InlineData(0)]
[InlineData(-7)]
public void Non_positive_page_size_falls_back_to_the_default(int configured)
=> GatewayGalaxyHierarchySource.EffectivePageSize(configured).ShouldBe(GatewayGalaxyHierarchySource.DefaultPageSize);
/// <summary>A repeated page token is refused rather than walked forever.</summary>
[Fact]
public async Task A_repeated_page_token_is_refused()
{
var calls = 0;
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
{
calls++;
var reply = new DiscoverHierarchyReply { NextPageToken = "same" };
reply.Objects.Add(Obj(calls));
return Task.FromResult(reply);
}
var ex = await Should.ThrowAsync<InvalidOperationException>(
() => GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 10, CancellationToken.None));
ex.Message.ShouldContain("repeated page token");
calls.ShouldBe(2);
}
}
@@ -0,0 +1,60 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
/// <summary>The platform-only query the driver seeds its ScanState probe watcher from.</summary>
public sealed class GatewayGalaxyPlatformSourceTests
{
private static GalaxyObject Obj(string tag, params string[] chain)
{
var o = new GalaxyObject { TagName = tag };
o.TemplateChain.Add(chain);
return o;
}
/// <summary>Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered.</summary>
[Fact]
public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered()
{
var requests = new List<DiscoverHierarchyRequest>();
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
{
requests.Add(req);
var reply = new DiscoverHierarchyReply();
if (req.PageToken == "")
{
reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]);
reply.NextPageToken = "p2";
}
else
{
reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]);
}
return Task.FromResult(reply);
}
var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None);
names.ShouldBe(["Plat01", "Eng01"]);
requests.Count.ShouldBe(2);
requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false);
requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine"));
}
/// <summary>A gateway that strips the template chain is trusted to have filtered server-side.</summary>
[Fact]
public async Task Objects_without_a_template_chain_are_kept()
{
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
{
var reply = new DiscoverHierarchyReply();
reply.Objects.Add(new GalaxyObject { TagName = "Plat01" });
return Task.FromResult(reply);
}
(await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 0, CancellationToken.None)).ShouldBe(["Plat01"]);
}
}
@@ -0,0 +1,69 @@
using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
/// <summary>
/// Review 2026-10-21 G-6: <c>ProbeTimeoutSeconds</c> was authored and compared for equivalence but
/// never consumed — the probe ran on the caller's budget alone. It now caps the probe deadline
/// (it can shorten the caller's budget, never extend it).
/// </summary>
[Trait("Category", "Unit")]
public sealed class GalaxyDriverProbeTimeoutTests
{
/// <summary>The effective deadline is the shorter of the caller's budget and the configured cap.</summary>
[Theory]
[InlineData(10, 30, 10)] // caller shorter → caller wins
[InlineData(10, 2, 2)] // config shorter → config caps
[InlineData(10, 0, 10)] // unset config → caller
[InlineData(10, -1, 10)] // nonsense config → caller
[InlineData(0, 5, 1)] // no caller budget → 1 s floor beats a longer config
[InlineData(0, 0, 1)] // nothing → 1 s floor
public void EffectiveTimeout_is_the_shorter_of_caller_and_config(int callerSeconds, int configSeconds, int expectedSeconds)
=> GalaxyDriverProbe.EffectiveTimeout(TimeSpan.FromSeconds(callerSeconds), configSeconds)
.ShouldBe(TimeSpan.FromSeconds(expectedSeconds));
/// <summary>
/// Against a port that accepts TCP but never speaks gRPC, a 1 s configured cap returns long
/// before the caller's 20 s budget. The gRPC handshake itself does the waiting here, so this
/// exercises the real deadline path, not the pure helper.
/// </summary>
[Fact]
public async Task Configured_cap_shortens_the_probe_against_a_silent_listener()
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
try
{
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
var config = $$"""
{
"Gateway": { "Endpoint": "http://127.0.0.1:{{port}}", "ApiKeySecretRef": "dev:k", "UseTls": false },
"MxAccess": { "ClientName": "probe-timeout" },
"Repository": {},
"Reconnect": {},
"ProbeTimeoutSeconds": 1
}
""";
var sw = Stopwatch.StartNew();
var result = await new GalaxyDriverProbe().ProbeAsync(config, TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
sw.Stop();
result.Ok.ShouldBeFalse();
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), $"the 1 s configured cap must bound the probe (message: {result.Message})");
result.Message.ShouldNotBeNull();
(result.Message!.Contains("timed out", StringComparison.OrdinalIgnoreCase)
|| result.Message.Contains("DeadlineExceeded", StringComparison.OrdinalIgnoreCase)
|| result.Message.Contains("handshake failed", StringComparison.OrdinalIgnoreCase))
.ShouldBeTrue($"unexpected message: '{result.Message}'");
}
finally
{
listener.Stop();
}
}
}
@@ -0,0 +1,112 @@
using System.Runtime.CompilerServices;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Health;
/// <summary>
/// Review 2026-10-21 G-4: the watcher's probe handles must live in the driver's
/// <see cref="SubscriptionRegistry"/> under the reserved id, follow the platform set, be
/// rebound on <see cref="PerPlatformProbeWatcher.ResubscribeAsync"/>, and leave on stop.
/// </summary>
public sealed class PerPlatformProbeWatcherRegistryTests
{
private sealed class FakeSubscriber : IGalaxySubscriber
{
private int _next;
public List<int[]> Unsubscribes { get; } = [];
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
=> Task.FromResult<IReadOnlyList<SubscribeResult>>(
[.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = ++_next, WasSuccessful = true })]);
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
{
Unsubscribes.Add([.. itemHandles]);
return Task.CompletedTask;
}
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await Task.Delay(Timeout.Infinite, cancellationToken);
yield break;
}
}
private static (long Id, string Ref)[] Resolve(SubscriptionRegistry registry, int handle) =>
[.. registry.ResolveSubscribers(handle)];
/// <summary>Synced platforms resolve through the registry under the reserved id; removed ones stop resolving.</summary>
[Fact]
public async Task Sync_registers_probe_handles_under_the_reserved_id_and_tracks_removals()
{
var registry = new SubscriptionRegistry();
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue();
var bindings = watcher.CurrentBindings;
bindings.Count.ShouldBe(2);
foreach (var b in bindings)
{
Resolve(registry, b.ItemHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, b.FullReference)]);
}
var bHandle = bindings.Single(b => b.FullReference == "B.ScanState").ItemHandle;
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
Resolve(registry, bHandle).ShouldBeEmpty("a removed platform's handle must leave the fan-out map");
var aHandle = bindings.Single(b => b.FullReference == "A.ScanState").ItemHandle;
Resolve(registry, aHandle).ShouldHaveSingleItem("positive control: the kept platform still resolves");
}
/// <summary>Resubscribe replaces the handles and rebinds: old handles stop resolving, new ones resolve.</summary>
[Fact]
public async Task Resubscribe_rebinds_the_registry_to_the_fresh_handles()
{
var registry = new SubscriptionRegistry();
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
var oldHandle = watcher.CurrentBindings.Single().ItemHandle;
(await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
var newHandle = watcher.CurrentBindings.Single().ItemHandle;
newHandle.ShouldNotBe(oldHandle);
Resolve(registry, oldHandle).ShouldBeEmpty();
Resolve(registry, newHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, "A.ScanState")]);
registry.TrackedSubscriptionCount.ShouldBe(1);
}
/// <summary>StopAsync unsubscribes the live handles and removes the reserved entry.</summary>
[Fact]
public async Task StopAsync_unsubscribes_and_removes_the_reserved_entry()
{
var registry = new SubscriptionRegistry();
var subscriber = new FakeSubscriber();
var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
var handles = watcher.CurrentBindings.Select(b => b.ItemHandle).OrderBy(h => h).ToArray();
await watcher.StopAsync(CancellationToken.None);
subscriber.Unsubscribes.ShouldHaveSingleItem().OrderBy(h => h).ShouldBe(handles);
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse();
registry.TrackedItemHandleCount.ShouldBe(0);
await Should.NotThrowAsync(() => watcher.StopAsync(CancellationToken.None)); // idempotent
}
/// <summary>A watcher without a registry (the decoding-table unit tests) still works and registers nothing.</summary>
[Fact]
public async Task Without_a_registry_the_watcher_keeps_its_handles_private()
{
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator());
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
watcher.CurrentBindings.ShouldHaveSingleItem();
(await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
}
}
@@ -0,0 +1,17 @@
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
/// <summary>
/// Classes that assert on process-wide telemetry (an <c>ActivityListener</c> on the Galaxy
/// ActivitySource, a <c>MeterListener</c> on the Galaxy meter) run with parallelization disabled:
/// those listeners see every Galaxy span / measurement in the process, so any parallel class that
/// drives a traced subscriber or the data writer — including a driver's background probe seed that
/// outlives its test — pollutes their counts (F-FU6, observed flaky in lane F fix round 2).
/// </summary>
[CollectionDefinition(Name, DisableParallelization = true)]
public sealed class ProcessWideTelemetryCollection
{
/// <summary>The collection name.</summary>
public const string Name = "Process-wide Galaxy telemetry";
}
@@ -0,0 +1,96 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Review 2026-10-21 G-10: a dispatch-loop exit left the read loop draining into a channel
/// nobody read, with health Healthy and nothing telling the supervisor — a clean termination
/// evading the watchdog. The exit now reports a stream fault. The fault is injected through
/// the pump's handle factory, the one dispatch-side seam that runs outside the per-handler
/// isolation.
/// </summary>
public sealed class EventPumpDispatchFaultTests
{
private const int WaitMs = 10_000;
private sealed class ChannelSubscriber : IGalaxySubscriber
{
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
=> Task.FromResult<IReadOnlyList<SubscribeResult>>([]);
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask EmitAsync(int handle) => _events.Writer.WriteAsync(new MxEvent
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { DoubleValue = 1 },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
});
}
/// <summary>A dispatch-loop exception reports a stream fault carrying that exception.</summary>
[Fact]
public async Task Dispatch_loop_exit_reports_a_stream_fault()
{
var subscriber = new ChannelSubscriber();
var registry = new SubscriptionRegistry();
registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
var fault = new TaskCompletionSource<Exception>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var pump = new EventPump(
subscriber, registry,
handleFactory: _ => throw new InvalidDataException("synthetic dispatch failure"),
channelCapacity: 4, clientName: "DispatchFault",
onStreamFault: ex => fault.TrySetResult(ex));
pump.Start();
await subscriber.EmitAsync(7);
(await Task.WhenAny(fault.Task, Task.Delay(WaitMs))).ShouldBe(fault.Task,
"a dispatch-loop exit must be reported as a stream fault");
(await fault.Task).ShouldBeOfType<InvalidDataException>();
}
/// <summary>Positive control: the same event with a working handle factory dispatches and reports no fault.</summary>
[Fact]
public async Task Same_event_with_a_working_factory_dispatches_without_a_fault()
{
var subscriber = new ChannelSubscriber();
var registry = new SubscriptionRegistry();
registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
var faulted = false;
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var pump = new EventPump(
subscriber, registry, channelCapacity: 4, clientName: "DispatchFault",
onStreamFault: _ => faulted = true);
pump.OnDataChange += (_, e) => delivered.TrySetResult(e);
pump.Start();
await subscriber.EmitAsync(7);
(await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task);
(await delivered.Task).FullReference.ShouldBe("Tank01.Level");
faulted.ShouldBeFalse();
}
}
@@ -0,0 +1,406 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Lane F fix round 1 (review of #735): the probe watcher is seeded at <c>InitializeAsync</c>
/// and re-seeded on rediscovery (the runtime never calls <c>DiscoverAsync</c> on a live driver);
/// a <c>SubscribeAsync</c> landing mid-teardown cannot create a pump on the dying subscriber; a
/// gateway close that never answers cannot park <c>ShutdownAsync</c>; a re-subscribe that
/// straddles a teardown never rebinds into the next runtime's registry.
/// </summary>
public sealed class GalaxyDriverFixRound1Tests
{
internal const int WaitMs = 10_000;
internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("FixRound1"),
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
new GalaxyReconnectOptions());
// ───────────────────────────── fakes ─────────────────────────────
private sealed class ScriptedPlatformSource : IGalaxyPlatformSource
{
private int _calls;
public int Calls => Volatile.Read(ref _calls);
public Func<int, IReadOnlyList<string>> Script { get; init; } = _ => ["Plat01"];
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
=> Task.FromResult(Script(Interlocked.Increment(ref _calls)));
}
internal sealed class ParkableSubscriber : IGalaxySubscriber
{
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
private int _nextHandle;
private int _subscribeCalls;
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
public List<string> Subscribed { get; } = [];
public int SubscribeCalls => Volatile.Read(ref _subscribeCalls);
/// <summary>When set, UnsubscribeBulk parks until <see cref="ReleaseUnsubscribe"/>.</summary>
public bool ParkUnsubscribe { get; set; }
public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult();
/// <summary>When > 0, that SubscribeBulk call (1-based) parks until <see cref="ReleaseSubscribe"/>.</summary>
public int ParkSubscribeCall { get; set; }
public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult();
public async Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
var call = Interlocked.Increment(ref _subscribeCalls);
if (call == ParkSubscribeCall)
{
SubscribeEntered.TrySetResult();
await _releaseSubscribe.Task.WaitAsync(cancellationToken);
}
var results = new List<SubscribeResult>();
lock (Subscribed)
{
foreach (var r in fullReferences)
{
var h = Interlocked.Increment(ref _nextHandle);
Subscribed.Add(r);
HandleByRef[r] = h;
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
}
}
return results;
}
/// <summary>Every handle passed to UnsubscribeBulk, in call order.</summary>
public List<int> Unsubscribed { get; } = [];
public async Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
{
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
if (ParkUnsubscribe)
{
UnsubscribeEntered.TrySetResult();
await _releaseUnsubscribe.Task.WaitAsync(cancellationToken);
}
}
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { BoolValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
});
}
internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource
{
public Channel<DeployEvent> Events { get; } = Channel.CreateUnbounded<DeployEvent>();
public int Opened;
public async IAsyncEnumerable<DeployEvent> WatchAsync(
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
{
Interlocked.Increment(ref Opened);
await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent
{
Sequence = sequence,
TimeOfLastDeployPresent = true,
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
});
}
internal static async Task WaitUntilAsync(Func<bool> condition, string because)
{
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
await Task.Delay(10);
}
}
internal static HostState? StateOf(GalaxyDriver driver, string host) =>
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
// ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ─────────────────────────────
/// <summary>A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses.</summary>
[Fact]
public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync()
{
var sub = new ParkableSubscriber();
var platforms = new ScriptedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
await driver.InitializeAsync("{}", CancellationToken.None);
// The seed runs in the background (fix round 2, N1) — init does not wait on it.
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the probe watcher must be seeded after init");
platforms.Calls.ShouldBe(1);
lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState");
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
"a ScanState event must surface the seeded platform on GetHostStatuses");
}
/// <summary>A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds.</summary>
[Fact]
public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds()
{
var sub = new ParkableSubscriber();
var deploys = new ChannelDeployWatchSource();
var platforms = new ScriptedPlatformSource
{
Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"],
};
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
var raised = 0;
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
await driver.InitializeAsync("{}", CancellationToken.None);
driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init");
await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run");
sub.Subscribed.ShouldBeEmpty();
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed)
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy
await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery");
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
"the rediscovery must re-seed the probe set");
platforms.Calls.ShouldBe(2);
sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true);
}
private sealed class ParkingPlatformSource : IGalaxyPlatformSource
{
private int _calls;
public int Calls => Volatile.Read(ref _calls);
public int ParkCall { get; init; }
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _release.TrySetResult();
public async Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
{
var call = Interlocked.Increment(ref _calls);
if (call == ParkCall)
{
Entered.TrySetResult();
await _release.Task.WaitAsync(cancellationToken);
return ["Plat01"]; // the pre-deploy view this pass fetched
}
return call == 1 ? [] : ["Plat01", "Eng02"];
}
}
/// <summary>
/// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts
/// after it — otherwise the probe set would stay on the stale view until the next deploy.
/// </summary>
[Fact]
public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass()
{
var sub = new ParkableSubscriber();
var deploys = new ChannelDeployWatchSource();
var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
var raised = 0;
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
try
{
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
(await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task,
"the first rediscovery pass must be parked mid-fetch");
await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero));
await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery");
}
finally
{
platforms.Release();
}
await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed");
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
"the final probe set must reflect the latest deploy, not the parked pass's stale view");
}
/// <summary>The production path (owned runtime) queries the platform source at init, with no DiscoverAsync.</summary>
[Fact]
public async Task Production_path_queries_the_platform_source_at_init()
{
var platforms = new ScriptedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms)
{
SessionOpenOverrideForTests = _ => Task.CompletedTask,
};
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init");
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
}
// ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ─────────────────────────────
/// <summary>
/// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before
/// the subscriber is released) is refused and leaves no pump; before the shutdown the same
/// call works (positive control).
/// </summary>
[Fact]
public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump()
{
var sub = new ParkableSubscriber();
var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the (background) seed must give the probe watcher its handle first");
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
sub.ParkUnsubscribe = true;
var shutdown = driver.ShutdownAsync(CancellationToken.None);
var subscribeCallsBefore = sub.SubscribeCalls;
try
{
(await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task,
"the teardown must be parked inside the probe watcher's unsubscribe");
shutdown.IsCompleted.ShouldBeFalse();
// The pump is still live here (the teardown takes it after the probe watcher) — reusing it
// is exactly the race: the call must be refused, not served on the dying subscriber.
driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken");
await Should.ThrowAsync<InvalidOperationException>(
() => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None));
sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway");
}
finally
{
// Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs.
sub.ReleaseUnsubscribe();
}
await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown");
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
await driver.InitializeAsync("{}", CancellationToken.None);
driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
}
// ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
/// <summary>A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline.</summary>
[Fact]
public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers()
{
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource())
{
SessionOpenOverrideForTests = _ => Task.CompletedTask,
SessionCloseDeadline = TimeSpan.FromMilliseconds(300),
};
await driver.InitializeAsync("{}", CancellationToken.None);
var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var closeToken = CancellationToken.None;
driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct =>
{
closeToken = ct;
closeEntered.TrySetResult();
await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway
};
var sw = System.Diagnostics.Stopwatch.StartNew();
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
sw.Stop();
closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted");
closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline");
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s");
driver.OwnedMxSessionForTests.ShouldBeNull();
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
}
// ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ─────────────────────────────
/// <summary>
/// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared
/// registry; the same replay without a teardown rebinds (positive control).
/// </summary>
[Fact]
public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind()
{
// Positive control first.
{
var sub = new ParkableSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
await driver.InvokeReplayForTestAsync(CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription");
sub.SubscribeCalls.ShouldBe(2);
}
{
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
var replay = driver.InvokeReplayForTestAsync(CancellationToken.None);
(await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task,
"the replay must be parked inside its SubscribeBulk");
await driver.ShutdownAsync(CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
sub.ReleaseSubscribe();
await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles");
}
}
}
@@ -0,0 +1,313 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Lane F fix round 2 (re-review of #735):
/// <list type="bullet">
/// <item>N1 — the init platform seed runs in the background: a slow or cancelled repository
/// fetch can neither delay nor fail <c>InitializeAsync</c> (Galaxy's Tier-A Initialize budget is
/// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown
/// is discarded.</item>
/// <item>N2 — a seed requested under a new runtime always gets a pass of its own, even when a
/// worker from the previous runtime is still running.</item>
/// <item>N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails
/// and unsubscribes instead of registering into the next runtime's registry.</item>
/// <item>The probe watcher's own subscribe re-checks its stop after the await.</item>
/// </list>
/// </summary>
public sealed class GalaxyDriverFixRound2Tests
{
private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2);
/// <summary>A platform source whose fetches park until released (optionally ignoring cancellation).</summary>
private sealed class GatedPlatformSource : IGalaxyPlatformSource
{
private int _calls;
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
public int Calls => Volatile.Read(ref _calls);
public int ParkCall { get; init; } = 1;
public bool IgnoreCancellation { get; init; }
public Func<int, IReadOnlyList<string>> Result { get; init; } = _ => ["Plat01"];
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource<CancellationToken> ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _release.TrySetResult();
public async Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
{
var call = Interlocked.Increment(ref _calls);
if (call == ParkCall)
{
ParkedToken.TrySetResult(cancellationToken);
Entered.TrySetResult();
if (IgnoreCancellation) await _release.Task;
else await _release.Task.WaitAsync(cancellationToken);
}
return Result(call);
}
}
private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource
{
private int _calls;
public int Calls => Volatile.Read(ref _calls);
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _calls);
return Task.FromException<IReadOnlyList<string>>(ex);
}
}
private static async Task Entered(TaskCompletionSource tcs, string because) =>
(await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because);
// ───────────────────────────── N1: background init seed ─────────────────────────────
/// <summary>
/// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy
/// inside its budget, and the teardown cancels the parked fetch.
/// </summary>
[Fact]
public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes()
{
var sub = new ParkableSubscriber();
var platforms = new GatedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
try
{
var sw = System.Diagnostics.Stopwatch.StartNew();
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
sw.Stop();
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
await Entered(platforms.Entered, "positive control: the seed fetch was started");
sw.Elapsed.ShouldBeLessThan(InitBudget);
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
var token = await platforms.ParkedToken.Task;
token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight");
}
finally
{
platforms.Release();
}
}
/// <summary>A fetch that is cancelled (the gateway's own deadline) does not fail init.</summary>
[Fact]
public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync()
{
var sub = new ParkableSubscriber();
var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline"));
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran");
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
// The driver keeps serving after the failed seed.
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(1);
}
/// <summary>
/// The production failure mode: the host's Tier-A Initialize timeout cancels the init token
/// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and
/// the seed — tied to the runtime, not to the init call — must survive that cancellation and
/// still land.
/// </summary>
[Fact]
public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed()
{
var sub = new ParkableSubscriber();
var platforms = new GatedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
using var initCts = new CancellationTokenSource();
try
{
initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk
await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget);
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
await Entered(platforms.Entered, "the seed fetch must be in flight");
await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked");
(await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse(
"the seed belongs to the runtime; the init call's timeout must not cancel it");
}
finally
{
platforms.Release();
}
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the seed must still land after the init token was cancelled");
}
/// <summary>Positive control: a fetch that completes after init still populates host statuses.</summary>
[Fact]
public async Task A_fetch_completing_after_init_populates_host_statuses()
{
var sub = new ParkableSubscriber();
var platforms = new GatedPlatformSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
try
{
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
await Entered(platforms.Entered, "the seed must be parked");
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
}
finally
{
platforms.Release();
}
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the late seed must reach the probe watcher");
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
"a ScanState event must surface the late-seeded platform");
}
// ───────────────────────────── N1 generation + N2: straddling worker ─────────────────────────────
/// <summary>
/// A seed from the previous runtime that completes after a teardown + re-init is discarded,
/// and the new runtime's own seed request still gets its own pass.
/// </summary>
[Fact]
public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded()
{
var sub = new ParkableSubscriber();
var platforms = new GatedPlatformSource
{
IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it
Result = call => call == 1 ? ["Stale01"] : ["Plat01"],
};
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
try
{
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
await Entered(platforms.Entered, "runtime 1's seed must be parked");
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
}
finally
{
platforms.Release();
}
await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass");
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
"the new runtime must be seeded");
driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]);
lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded");
}
// ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ─────────────────────────────
/// <summary>
/// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry
/// back into the registry, and must release the handles it just acquired. Positive control:
/// the same sync without a stop registers.
/// </summary>
[Fact]
public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry()
{
{
var registry = new SubscriptionRegistry();
using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry);
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control");
}
{
var registry = new SubscriptionRegistry();
var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry);
var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk");
await watcher.StopAsync(CancellationToken.None);
sub.ReleaseSubscribe();
await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register");
watcher.WatchedPlatforms.ShouldBeEmpty();
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"],
"the handle acquired after the stop must be released");
}
}
// ───────────────────────────── N3: subscribe / read straddling a teardown ─────────────────────────────
/// <summary>
/// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing
/// and unsubscribes its handles; a subscribe on the live runtime registers (positive control).
/// </summary>
[Fact]
public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing()
{
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers");
var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None);
try
{
await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk");
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
}
finally
{
sub.ReleaseSubscribe();
}
await Should.ThrowAsync<InvalidOperationException>(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry");
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]);
await driver.InitializeAsync("{}", CancellationToken.None);
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry");
}
/// <summary>The subscribe-once read path: same straddle, same outcome.</summary>
[Fact]
public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing()
{
var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None);
try
{
await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk");
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
}
finally
{
sub.ReleaseSubscribe();
}
await Should.ThrowAsync<InvalidOperationException>(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]);
}
}
@@ -0,0 +1,217 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Review 2026-10-21 G-4 / G-7 / G-11: the per-platform <c>ScanState</c> probing was structurally
/// dead (its handles were never in the registry the pump resolves through, and a reconnect
/// never re-subscribed them), discovery fetched the full galaxy twice, and
/// <c>LastSuccessfulRead</c> never moved on data. These drive the real driver → pump → watcher
/// → aggregator path with a fake subscriber and a fake hierarchy.
/// </summary>
public sealed class GalaxyDriverProbeWiringTests
{
private const int WaitMs = 10_000;
private static GalaxyDriverOptions Opts() => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("ProbeWiring"),
new GalaxyRepositoryOptions(WatchDeployEvents: false),
new GalaxyReconnectOptions());
private sealed class CountingHierarchySource : IGalaxyHierarchySource
{
public int Calls;
public Task<IReadOnlyList<GalaxyObject>> GetHierarchyAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref Calls);
var platform = new GalaxyObject { GobjectId = 1, TagName = "Plat01", ContainedName = "Plat01" };
platform.TemplateChain.Add("$WinPlatform");
var tank = new GalaxyObject { GobjectId = 2, TagName = "Tank01", ContainedName = "Tank01", ParentGobjectId = 1 };
tank.TemplateChain.Add("$UserDefined");
tank.Attributes.Add(new GalaxyAttribute { AttributeName = "Level", FullTagReference = "Tank01.Level", MxDataType = 5 });
return Task.FromResult<IReadOnlyList<GalaxyObject>>([platform, tank]);
}
}
private sealed class StreamingSubscriber : IGalaxySubscriber
{
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
private int _nextHandle;
public List<string> Subscribed { get; } = [];
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
var results = new List<SubscribeResult>();
lock (Subscribed)
{
foreach (var r in fullReferences)
{
var h = Interlocked.Increment(ref _nextHandle);
Subscribed.Add(r);
HandleByRef[r] = h;
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
}
}
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
}
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { BoolValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
});
public ValueTask EmitDoubleAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { DoubleValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
});
}
private sealed class NullBuilder : IAddressSpaceBuilder
{
public IAddressSpaceBuilder Folder(string browseName, string displayName) => this;
public IVariableHandle Variable(string browseName, string displayName, DriverAttributeInfo attributeInfo) => new Handle(attributeInfo.FullName);
public void AddProperty(string browseName, DriverDataType dataType, object? value) { }
private sealed class Handle(string fullReference) : IVariableHandle
{
public string FullReference { get; } = fullReference;
public IAlarmConditionSink MarkAsAlarmCondition(AlarmConditionInfo alarmInfo) => new NoopSink();
}
private sealed class NoopSink : IAlarmConditionSink
{
public void OnTransition(AlarmEventArgs args) { }
}
}
private static async Task WaitUntilAsync(Func<bool> condition, string because)
{
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
await Task.Delay(10);
}
}
private static HostState? StateOf(GalaxyDriver driver, string host) =>
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
/// <summary>
/// A ScanState event on the probe handle updates GetHostStatuses; after a reconnect replay
/// the probe is re-subscribed and events on the NEW handle keep updating it, while the dead
/// pre-reconnect handle no longer does. Probe traffic never reaches public OnDataChange.
/// </summary>
[Fact]
public async Task ScanState_event_updates_GetHostStatuses_and_survives_a_reconnect()
{
var sub = new StreamingSubscriber();
var source = new CountingHierarchySource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
var publicRefs = new List<string>();
driver.OnDataChange += (_, e) => { lock (publicRefs) publicRefs.Add(e.FullReference); };
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
// The pump starts on the first value subscription (the probe watcher does not start it).
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
var probeHandle = sub.HandleByRef["Plat01.ScanState"];
await sub.EmitBoolAsync(probeHandle, true);
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
"a ScanState=true event must surface Plat01 as Running");
// Reconnect replay: every pre-reconnect handle is dead; the probe must rebind.
await driver.InvokeReplayForTestAsync(CancellationToken.None);
var rebound = sub.HandleByRef["Plat01.ScanState"];
rebound.ShouldNotBe(probeHandle, "replay must re-subscribe the probe and record the fresh handle");
await sub.EmitBoolAsync(rebound, false);
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Stopped,
"a ScanState=false event on the post-reconnect handle must surface Plat01 as Stopped");
// The dead handle is no longer routed (positive control: the live one still is).
await sub.EmitBoolAsync(probeHandle, true);
await sub.EmitDoubleAsync(sub.HandleByRef["Tank01.Level"], 1.0);
await WaitUntilAsync(() => { lock (publicRefs) return publicRefs.Contains("Tank01.Level"); }, "value traffic must still flow");
StateOf(driver, "Plat01").ShouldBe(HostState.Stopped, "an event on the pre-reconnect handle must be dropped");
lock (publicRefs) publicRefs.ShouldNotContain("Plat01.ScanState", "probe traffic is the watcher's, never the host's");
}
/// <summary>Discovery asks the gateway for the hierarchy once; the probe set is selected from that same sweep.</summary>
[Fact]
public async Task DiscoverAsync_fetches_the_hierarchy_once_and_still_syncs_the_probe_set()
{
var sub = new StreamingSubscriber();
var source = new CountingHierarchySource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
source.Calls.ShouldBe(1, "the probe selection must reuse the discoverer's sweep");
sub.Subscribed.ShouldContain("Plat01.ScanState", "positive control: the platform probe was still synced");
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
}
/// <summary>Inbound stream data advances LastSuccessfulRead (throttled); the throttle holds the stamp still.</summary>
[Fact]
public async Task Stream_data_advances_LastSuccessfulRead_under_a_throttle()
{
var sub = new StreamingSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
{
LivenessStampThrottle = TimeSpan.FromHours(1),
};
await driver.InitializeAsync("{}", CancellationToken.None);
var initStamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
await Task.Delay(20);
var delivered = 0;
driver.OnDataChange += (_, _) => Interlocked.Increment(ref delivered);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
var handle = sub.HandleByRef["Tank01.Level"];
await sub.EmitDoubleAsync(handle, 1.0);
await WaitUntilAsync(() => Volatile.Read(ref delivered) == 1, "first event must be delivered");
var first = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
first.ShouldBeGreaterThan(initStamp, "data on the stream is liveness evidence");
await Task.Delay(20);
await sub.EmitDoubleAsync(handle, 2.0);
await WaitUntilAsync(() => Volatile.Read(ref delivered) == 2, "second event must be delivered");
driver.GetHealth().LastSuccessfulRead.ShouldBe(first, "inside the throttle window the stamp holds still");
}
}
@@ -0,0 +1,386 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// <see cref="GalaxyDriver.ShutdownAsync"/> used to be a no-op (review 2026-10-21, G-1/G-2/G-9):
/// the gRPC runtime outlived the driver on every respawn, removal and <c>/raw</c> browse, and
/// the host's comms-loss recovery (<c>ShutdownAsync</c> → <c>InitializeAsync</c> on the SAME
/// object) neither tore down nor rebuilt anything. These tests pin the teardown at every
/// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed,
/// the owned channels + session — and the rebuild on the next <c>InitializeAsync</c>.
/// </summary>
public sealed class GalaxyDriverShutdownTests
{
private const int WaitMs = 10_000;
private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("ShutdownTest"),
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
new GalaxyReconnectOptions());
// ───────────────────────────── fakes ─────────────────────────────
/// <summary>
/// Subscriber whose event stream records the token it was opened with and completes a
/// signal when that token fires — the observable for "the StreamEvents stream was cancelled".
/// </summary>
private sealed class StreamingSubscriber : IGalaxySubscriber
{
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
private int _nextHandle;
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
public List<int> Unsubscribed { get; } = [];
public int StreamsOpened;
public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
var results = new List<SubscribeResult>();
foreach (var r in fullReferences)
{
var h = Interlocked.Increment(ref _nextHandle);
HandleByRef[r] = h;
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
}
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
}
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
{
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
return Task.CompletedTask;
}
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
Interlocked.Increment(ref StreamsOpened);
using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult());
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { DoubleValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
});
}
/// <summary>
/// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when
/// the enumerator is cancelled or completes) — the observable for "exactly one watcher".
/// </summary>
private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource
{
private int _open;
private int _opened;
public Channel<DeployEvent> Current { get; private set; } = Channel.CreateUnbounded<DeployEvent>();
public int Open => Volatile.Read(ref _open);
public int Opened => Volatile.Read(ref _opened);
public async IAsyncEnumerable<DeployEvent> WatchAsync(
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
{
Interlocked.Increment(ref _open);
Interlocked.Increment(ref _opened);
var channel = Current = Channel.CreateUnbounded<DeployEvent>();
try
{
await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
finally
{
Interlocked.Decrement(ref _open);
}
}
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent
{
Sequence = sequence,
TimeOfLastDeployPresent = true,
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
});
}
/// <summary>Canned platform set so a production-path init never dials the (fake) gateway for one.</summary>
private sealed class NoPlatforms : IGalaxyPlatformSource
{
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
=> Task.FromResult<IReadOnlyList<string>>([]);
}
private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed
{
public int Starts;
public int Disposes;
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
public void Start() => Starts++;
public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition(
"Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise,
AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi"));
public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; }
}
private static async Task WaitUntilAsync(Func<bool> condition, string because)
{
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
await Task.Delay(10);
}
}
// ───────────────────────────── G-1: streams + pump ─────────────────────────────
/// <summary>
/// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the
/// registry is empty. Positive control: an event delivered through that same stream
/// reached OnDataChange before the shutdown.
/// </summary>
[Fact]
public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump()
{
var sub = new StreamingSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
driver.OnDataChange += (_, e) => delivered.TrySetResult(e);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
driver.HasEventPumpForTests.ShouldBeTrue();
await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0);
(await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task,
"positive control: the stream must deliver before the shutdown");
await driver.ShutdownAsync(CancellationToken.None);
// Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the
// enumerator's cancellation registration has run by the time it returns (observed flaky).
(await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task,
"ShutdownAsync must cancel the StreamEvents stream");
driver.HasEventPumpForTests.ShouldBeFalse();
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init");
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown");
}
/// <summary>
/// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT
/// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive
/// control on both sides of the shutdown.
/// </summary>
[Fact]
public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it()
{
var feed = new RecordingAlarmFeed();
var sub = new StreamingSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
subscriber: sub, alarmFeed: feed);
var received = 0;
driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
feed.Emit();
received.ShouldBe(1, "positive control: the feed is wired before the shutdown");
await driver.ShutdownAsync(CancellationToken.None);
feed.Emit();
received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent");
feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector");
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
feed.Emit();
received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed");
feed.Starts.ShouldBe(2);
}
// ───────────────────────────── G-2: one DeployWatcher ─────────────────────────────
/// <summary>
/// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch
/// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly
/// once — five stacked watchers would raise five times.
/// </summary>
[Fact]
public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher()
{
var source = new CountingDeployWatchSource();
var sub = new StreamingSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
deployWatchSource: source);
var raised = 0;
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
await driver.InitializeAsync("{}", CancellationToken.None);
for (var cycle = 0; cycle < 5; cycle++)
{
// Let this cycle's watch stream actually open before the next shutdown, so every cycle
// holds a live stream the teardown has to cancel (a loop cancelled before its first
// open would make the count vacuous).
await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open");
await driver.ShutdownAsync(CancellationToken.None);
await driver.InitializeAsync("{}", CancellationToken.None);
}
await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1,
$"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}");
driver.DeployWatcherForTests.ShouldNotBeNull();
// Bootstrap (suppressed) then a genuine change.
await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery");
await Task.Delay(200);
Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy");
await driver.ShutdownAsync(CancellationToken.None);
await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream");
driver.DeployWatcherForTests.ShouldBeNull();
}
/// <summary>A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one.</summary>
[Fact]
public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher()
{
var source = new CountingDeployWatchSource();
var sub = new StreamingSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
deployWatchSource: source);
await driver.InitializeAsync("{}", CancellationToken.None);
var first = driver.DeployWatcherForTests;
await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open");
await driver.InitializeAsync("{}", CancellationToken.None);
await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1,
$"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}");
driver.DeployWatcherForTests.ShouldNotBeSameAs(first);
}
// ───────────────────────────── G-1/G-9: owned runtime, channels + session ─────────────────────────────
/// <summary>
/// The production path (no injected seams): InitializeAsync builds the owned client, session
/// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the
/// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's
/// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC
/// channels are real but lazy (nothing dials until a call is made).
/// </summary>
[Fact]
public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds()
{
var opens = 0;
var source = new CountingDeployWatchSource();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source,
platformSource: new NoPlatforms())
{
SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; },
};
await driver.InitializeAsync("{}", CancellationToken.None);
opens.ShouldBe(1);
var client = driver.OwnedMxClientForTests.ShouldNotBeNull();
var session = driver.OwnedMxSessionForTests.ShouldNotBeNull();
session.IsConnected.ShouldBeTrue();
driver.SupervisorForTests.ShouldNotBeNull();
driver.ProbeWatcherForTests.ShouldNotBeNull();
driver.DeployWatcherForTests.ShouldNotBeNull();
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
await driver.ShutdownAsync(CancellationToken.None);
session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session");
await Should.ThrowAsync<ObjectDisposedException>(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None),
"the owned gateway channel must be disposed");
driver.OwnedMxClientForTests.ShouldBeNull();
driver.OwnedMxSessionForTests.ShouldBeNull();
driver.SupervisorForTests.ShouldBeNull();
driver.ProbeWatcherForTests.ShouldBeNull();
driver.DeployWatcherForTests.ShouldBeNull();
await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled");
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp);
// The watchdog's second half: a fresh runtime on the same object.
await driver.InitializeAsync("{}", CancellationToken.None);
opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one");
driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client);
driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session);
driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue();
driver.SupervisorForTests.ShouldNotBeNull();
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher");
}
/// <summary>
/// A connect that fails mid-build releases the partial runtime (no orphaned client per
/// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing.
/// </summary>
[Fact]
public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds()
{
var attempts = 0;
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
platformSource: new NoPlatforms())
{
SessionOpenOverrideForTests = _ => ++attempts == 1
? throw new InvalidOperationException("gateway refused")
: Task.CompletedTask,
};
var ex = await Should.ThrowAsync<InvalidOperationException>(() => driver.InitializeAsync("{}", CancellationToken.None));
ex.Message.ShouldBe("gateway refused");
driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client");
driver.OwnedMxSessionForTests.ShouldBeNull();
driver.GetHealth().State.ShouldBe(DriverState.Faulted);
driver.GetHealth().LastError.ShouldBe("gateway refused");
await driver.InitializeAsync("{}", CancellationToken.None);
attempts.ShouldBe(2);
driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue();
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
}
/// <summary>DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op.</summary>
[Fact]
public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless()
{
var sub = new StreamingSubscriber();
var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
await driver.ShutdownAsync(CancellationToken.None);
await Should.NotThrowAsync(async () => await driver.DisposeAsync());
await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None));
await Should.ThrowAsync<ObjectDisposedException>(() => driver.InitializeAsync("{}", CancellationToken.None));
}
}
@@ -0,0 +1,90 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// <see cref="GalaxyMxSession.CloseAsync"/> bounds the gateway <c>CloseSession</c> round-trip by
/// the caller's token (the SDK dispose closes under its 30 s call timeout) and leaves the session
/// re-openable.
/// </summary>
public sealed class GalaxyMxSessionCloseTests
{
/// <summary>A close against a never-answering gateway returns when the token fires, without throwing.</summary>
[Fact]
public async Task CloseAsync_returns_at_the_token_deadline_against_a_never_answering_gateway()
{
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
{
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
CloseOverrideForTests = ct => Task.Delay(Timeout.Infinite, ct),
};
await session.ConnectAsync(null!, CancellationToken.None);
session.IsConnected.ShouldBeTrue();
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
var sw = System.Diagnostics.Stopwatch.StartNew();
await session.CloseAsync(cts.Token).WaitAsync(TimeSpan.FromSeconds(10));
sw.Stop();
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5));
session.IsConnected.ShouldBeFalse();
// Still usable: a close is not a dispose.
await session.ConnectAsync(null!, CancellationToken.None);
session.IsConnected.ShouldBeTrue();
}
/// <summary>Positive control: a close that answers completes and reports the token it was handed.</summary>
[Fact]
public async Task CloseAsync_completes_when_the_gateway_answers()
{
CancellationToken seen = default;
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
{
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
CloseOverrideForTests = ct => { seen = ct; return Task.CompletedTask; },
};
await session.ConnectAsync(null!, CancellationToken.None);
using var cts = new CancellationTokenSource();
await session.CloseAsync(cts.Token);
seen.ShouldBe(cts.Token, "the caller's token must reach the gateway round-trip");
session.IsConnected.ShouldBeFalse();
}
/// <summary>
/// The driver's teardown is CloseAsync (bounded) then DisposeAsync (final). The dispose must
/// not issue a second gateway close; a fresh connect after a close must close again (positive
/// control: the guard keys on "was connected", not on "ever closed").
/// </summary>
[Fact]
public async Task CloseAsync_then_DisposeAsync_closes_the_gateway_session_once()
{
var closes = 0;
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
{
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
};
await session.ConnectAsync(null!, CancellationToken.None);
await session.CloseAsync(CancellationToken.None);
await session.DisposeAsync();
closes.ShouldBe(1, "dispose after a close must not close the gateway session again");
var reopened = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
{
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
};
await reopened.ConnectAsync(null!, CancellationToken.None);
await reopened.CloseAsync(CancellationToken.None);
await reopened.ConnectAsync(null!, CancellationToken.None);
await reopened.DisposeAsync();
closes.ShouldBe(3, "each live session closes exactly once");
}
}
@@ -14,6 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <see cref="ActivityListener"/> rather than asserting on internal state, so the
/// tests double as documentation of the listener-side contract.
/// </summary>
[Collection(ProcessWideTelemetryCollection.Name)]
public sealed class GalaxyTelemetryTests
{
/// <summary>Subscribes an ActivityListener for the test, captures each spawned activity.</summary>
@@ -0,0 +1,170 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Review 2026-10-21 G-8: a batch of writes ran strictly serially, three round-trips per
/// first-touch tag. Distinct tags now run as concurrent lanes bounded at
/// <see cref="GatewayGalaxyDataWriter.MaxConcurrentWriteLanes"/>; same-tag writes keep their
/// order inside one lane; results stay at their request's index.
/// </summary>
public sealed class GatewayGalaxyDataWriterConcurrencyTests
{
private const int WaitMs = 10_000;
private static GatewayGalaxyDataWriter NewWriter()
=> new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0);
/// <summary>
/// Fake ops whose AddItem parks every caller until <see cref="ReleaseAt"/> callers are in
/// flight at once (or a safety delay elapses), recording the peak concurrency.
/// </summary>
private sealed class GatingOps : IMxWriteOps
{
private int _inFlight;
private int _peak;
private int _nextHandle;
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly ConcurrentDictionary<string, int> _handles = new(StringComparer.OrdinalIgnoreCase);
public int ReleaseAt { get; init; } = int.MaxValue;
public int Peak => Volatile.Read(ref _peak);
public ConcurrentQueue<(int Handle, MxValue Value)> RawWrites { get; } = new();
public ConcurrentQueue<string> AddItems { get; } = new();
public Func<int, MxCommandReply>? WriteResponder { get; init; }
/// <summary>Deterministic handle per tag when set; otherwise handles are assigned in AddItem completion order.</summary>
public Func<string, int>? HandleFor { get; init; }
public string SessionId => "sess";
public int ServerHandle => 1;
public async Task<int> AddItemAsync(string fullRef, CancellationToken ct)
{
AddItems.Enqueue(fullRef);
var now = Interlocked.Increment(ref _inFlight);
int seen;
do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen);
if (now >= ReleaseAt) _release.TrySetResult();
// Park on the signal; the delay is only a safety net against a wedged test (a starved
// pool must not release the gate early and turn the peak assertion flaky).
await Task.WhenAny(_release.Task, Task.Delay(WaitMs, ct));
Interlocked.Decrement(ref _inFlight);
return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle));
}
public Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken ct)
=> Task.FromResult(new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } });
public Task<MxCommandReply> WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct)
{
RawWrites.Enqueue((itemHandle, value));
return Task.FromResult(WriteResponder?.Invoke(itemHandle) ?? GoodStatusRow());
}
// A reply carrying a Good MX status ROW (not an empty statuses array): the writer's static
// meters are process-wide, and an empty-statuses Good would increment galaxy.writes.unconfirmed
// under the meter tests running in parallel in another class.
private static MxCommandReply GoodStatusRow()
{
var reply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } };
reply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok });
return reply;
}
}
private static SecurityClassification Free(string _) => SecurityClassification.FreeAccess;
/// <summary>Sixteen distinct tags: lanes overlap (not serial) and never exceed the cap of eight.</summary>
[Fact]
public async Task Distinct_tags_write_concurrently_bounded_at_the_lane_cap()
{
var ops = new GatingOps { ReleaseAt = GatewayGalaxyDataWriter.MaxConcurrentWriteLanes };
var writes = Enumerable.Range(0, 16).Select(i => new WriteRequest($"Tag{i}.PV", (double)i)).ToArray();
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
ops.Peak.ShouldBe(GatewayGalaxyDataWriter.MaxConcurrentWriteLanes,
"first-touch lanes must overlap (serial = 1) and stay bounded (unbounded > 8)");
results.Count.ShouldBe(16);
results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
ops.AddItems.Count.ShouldBe(16, "one AddItem per distinct tag");
ops.RawWrites.Count.ShouldBe(16);
}
/// <summary>Writes to the same tag stay in input order inside their lane and share one AddItem.</summary>
[Fact]
public async Task Same_tag_writes_stay_ordered_and_share_one_handle()
{
var ops = new GatingOps { ReleaseAt = 2 };
var writes = new[]
{
new WriteRequest("Tank01.SP", 1.0),
new WriteRequest("Tank02.SP", 10.0),
new WriteRequest("Tank01.SP", 2.0),
new WriteRequest("Tank01.SP", 3.0),
};
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
results.Count.ShouldBe(4);
results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
ops.AddItems.Count(a => a == "Tank01.SP").ShouldBe(1, "a lane AddItems its tag once");
var tank01 = ops.RawWrites.Where(w => w.Handle == ops.RawWrites.First(x => x.Value.DoubleValue == 1.0).Handle)
.Select(w => w.Value.DoubleValue).ToArray();
tank01.ShouldBe([1.0, 2.0, 3.0], "same-tag writes must keep their authored order");
}
/// <summary>A per-tag failure lands at that request's index while the other lanes stay Good.</summary>
[Fact]
public async Task Results_land_at_their_request_index()
{
var ops = new GatingOps
{
ReleaseAt = 3,
HandleFor = tag => tag[0] - 'A' + 1, // A.PV → 1, B.PV → 2, C.PV → 3
WriteResponder = handle => new MxCommandReply
{
ProtocolStatus = new ProtocolStatus
{
Code = handle == 2 ? ProtocolStatusCode.MxaccessFailure : ProtocolStatusCode.Ok,
},
},
};
var writes = new[]
{
new WriteRequest("A.PV", 1.0),
new WriteRequest("B.PV", 2.0), // refused by the gateway
new WriteRequest("C.PV", 3.0),
};
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
results.Select(r => r.StatusCode).ShouldBe(
[StatusCodeMap.Good, StatusCodeMap.BadCommunicationError, StatusCodeMap.Good],
"each result must sit at its request's index regardless of lane completion order");
}
/// <summary>A single-tag batch takes the direct path and still produces one result per request.</summary>
[Fact]
public async Task Single_tag_batch_runs_serially_in_order()
{
var ops = new GatingOps { ReleaseAt = 1 };
var writes = new[] { new WriteRequest("X.PV", 1.0), new WriteRequest("X.PV", 2.0) };
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
results.Count.ShouldBe(2);
ops.Peak.ShouldBe(1);
ops.RawWrites.Select(w => w.Value.DoubleValue).ShouldBe([1.0, 2.0]);
}
}
@@ -13,6 +13,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal
/// <see cref="IMxWriteOps"/> seam (the SDK session types are sealed + internal-ctor and cannot be faked).
/// </summary>
[Collection(ProcessWideTelemetryCollection.Name)]
public sealed class GatewayGalaxyDataWriterFailClosedTests
{
private static GatewayGalaxyDataWriter NewWriter()
@@ -0,0 +1,92 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Review 2026-10-21 G-3: <see cref="ReconnectSupervisor.ReportTransportFailure"/> allocated a
/// new <see cref="CancellationTokenSource"/> per recovery cycle and only the last one was ever
/// disposed. Each cycle now releases its own CTS when its loop exits, and
/// <see cref="ReconnectSupervisor.StopAsync"/> awaits the loop instead of blocking on it.
/// </summary>
public sealed class ReconnectSupervisorCycleCtsTests
{
private const int WaitMs = 10_000;
private static ReconnectOptions FastBackoff() =>
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
/// <summary>Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control).</summary>
[Fact]
public async Task Every_completed_recovery_cycle_releases_its_own_cts()
{
var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var hold = new[] { false }; // read on the recovery loop's thread, written here — Volatile both ways
using var sup = new ReconnectSupervisor(
reopen: async _ =>
{
if (Volatile.Read(ref hold[0]))
{
reopenEntered.TrySetResult();
await holdReopen.Task;
}
},
replay: _ => Task.CompletedTask,
options: FastBackoff());
for (var i = 0; i < 3; i++)
{
sup.ReportTransportFailure(new IOException($"drop {i}"));
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
}
await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS");
sup.CyclesStarted.ShouldBe(3);
// Positive control: a cycle parked inside reopen still holds its CTS.
Volatile.Write(ref hold[0], true);
sup.ReportTransportFailure(new IOException("drop 3"));
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
sup.CyclesStarted.ShouldBe(4);
sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits");
holdReopen.TrySetResult();
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit");
}
/// <summary>StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking.</summary>
[Fact]
public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts()
{
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var sup = new ReconnectSupervisor(
reopen: async ct =>
{
reopenEntered.TrySetResult();
await Task.Delay(Timeout.Infinite, ct);
},
replay: _ => Task.CompletedTask,
options: FastBackoff());
sup.ReportTransportFailure(new IOException("drop"));
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
var stop = sup.StopAsync();
(await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return");
sup.CyclesReleased.ShouldBe(1);
await Should.NotThrowAsync(sup.StopAsync); // idempotent
Should.NotThrow(sup.Dispose);
}
private static async Task WaitUntilAsync(Func<bool> condition, string because)
{
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
await Task.Delay(10);
}
}
}