Merge lane F: Galaxy driver teardown, probing, knobs (#735)
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -61,8 +61,12 @@ public sealed class GalaxyDiscoverer
|
||||
/// </summary>
|
||||
/// <param name="builder">The address space builder to populate with discovery results.</param>
|
||||
/// <param name="cancellationToken">The cancellation token for the operation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken)
|
||||
/// <returns>
|
||||
/// The hierarchy this pass fetched, so a caller that needs the object set for something
|
||||
/// else (the driver selects its <c>$WinPlatform</c> / <c>$AppEngine</c> probe targets from
|
||||
/// it) reuses the one sweep instead of asking the gateway for the full galaxy twice.
|
||||
/// </returns>
|
||||
public async Task<IReadOnlyList<GalaxyObject>> DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
var objects = await _source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false);
|
||||
@@ -132,6 +136,8 @@ public sealed class GalaxyDiscoverer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return objects;
|
||||
}
|
||||
|
||||
// Resolve (and memoise) the folder for a gobject, recursively creating its parent's folder
|
||||
|
||||
@@ -5,22 +5,83 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
/// <summary>
|
||||
/// Default <see cref="IGalaxyHierarchySource"/> wrapping the gateway's
|
||||
/// <see cref="GalaxyRepositoryClient"/>. Pages internally via the client's overload.
|
||||
/// <see cref="GalaxyRepositoryClient"/>. Pages the <c>DiscoverHierarchy</c> RPC itself,
|
||||
/// at the configured <c>Repository.DiscoverPageSize</c>: the client's typed
|
||||
/// <c>DiscoverHierarchyAsync</c> overloads hard-code a 5000-row page
|
||||
/// (<c>DiscoverHierarchyOptions</c> carries no page size), so the knob the AdminUI
|
||||
/// authors and the factory parses used to reach nothing. The raw RPC takes the page
|
||||
/// size on the wire (<c>DiscoverHierarchyRequest.page_size</c>), so it is honoured here.
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyHierarchySource : IGalaxyHierarchySource
|
||||
{
|
||||
/// <summary>The page size used when the configured one is not positive — the client library's own default.</summary>
|
||||
public const int DefaultPageSize = 5000;
|
||||
|
||||
private readonly GalaxyRepositoryClient _client;
|
||||
private readonly int _pageSize;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="GatewayGalaxyHierarchySource"/> class.
|
||||
/// </summary>
|
||||
/// <param name="client">The gateway's Galaxy repository client for discovering the object hierarchy.</param>
|
||||
public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client)
|
||||
/// <param name="pageSize">
|
||||
/// Objects per <c>DiscoverHierarchy</c> page (<c>Repository.DiscoverPageSize</c>). A value that is
|
||||
/// not positive falls back to <see cref="DefaultPageSize"/>.
|
||||
/// </param>
|
||||
public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client, int pageSize = DefaultPageSize)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
_pageSize = EffectivePageSize(pageSize);
|
||||
}
|
||||
|
||||
/// <summary>The page size this source sends on every <c>DiscoverHierarchy</c> request.</summary>
|
||||
public int PageSize => _pageSize;
|
||||
|
||||
/// <summary>Clamp a configured page size to something the gateway accepts: non-positive → <see cref="DefaultPageSize"/>.</summary>
|
||||
/// <param name="configured">The configured value.</param>
|
||||
/// <returns>The page size to send.</returns>
|
||||
public static int EffectivePageSize(int configured) => configured > 0 ? configured : DefaultPageSize;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<GalaxyObject>> GetHierarchyAsync(CancellationToken cancellationToken)
|
||||
=> _client.DiscoverHierarchyAsync(cancellationToken);
|
||||
=> PageAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Walk every <c>DiscoverHierarchy</c> page at <paramref name="pageSize"/> and concatenate the
|
||||
/// objects. Mirrors the client library's own loop (including its repeated-page-token guard,
|
||||
/// which turns a misbehaving gateway into an error rather than an endless walk), with the
|
||||
/// RPC behind a delegate so the loop is unit-testable without a channel.
|
||||
/// </summary>
|
||||
/// <param name="discover">The raw <c>DiscoverHierarchy</c> RPC.</param>
|
||||
/// <param name="pageSize">Objects per page; sent on every request.</param>
|
||||
/// <param name="cancellationToken">Cancellation token for the walk.</param>
|
||||
/// <returns>Every object across all pages, in gateway order.</returns>
|
||||
internal static async Task<IReadOnlyList<GalaxyObject>> PageAsync(
|
||||
Func<DiscoverHierarchyRequest, CancellationToken, Task<DiscoverHierarchyReply>> discover,
|
||||
int pageSize,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(discover);
|
||||
var effective = EffectivePageSize(pageSize);
|
||||
|
||||
var objects = new List<GalaxyObject>();
|
||||
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
|
||||
var pageToken = string.Empty;
|
||||
do
|
||||
{
|
||||
var request = new DiscoverHierarchyRequest { PageSize = effective, PageToken = pageToken };
|
||||
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
objects.AddRange(reply.Objects);
|
||||
pageToken = reply.NextPageToken ?? string.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Galaxy DiscoverHierarchy returned a repeated page token '{pageToken}' — refusing to loop.");
|
||||
}
|
||||
}
|
||||
while (!string.IsNullOrWhiteSpace(pageToken));
|
||||
|
||||
return objects;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
using ZB.MOM.WW.MxGateway.Client;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
/// <summary>
|
||||
/// Default <see cref="IGalaxyPlatformSource"/>: a platform-only <c>DiscoverHierarchy</c> sweep
|
||||
/// over the gateway's <see cref="GalaxyRepositoryClient"/> — the request filters on
|
||||
/// <c>template_chain_contains</c> (<c>$WinPlatform</c>, <c>$AppEngine</c>) and asks for no
|
||||
/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the
|
||||
/// address-space discovery pulls. Paged at the configured <c>Repository.DiscoverPageSize</c>.
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
/// <summary>Template-chain tokens that identify a platform / engine object.</summary>
|
||||
public static readonly IReadOnlyList<string> PlatformTemplates = ["$WinPlatform", "$AppEngine"];
|
||||
|
||||
private readonly GalaxyRepositoryClient _client;
|
||||
private readonly int _pageSize;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="GatewayGalaxyPlatformSource"/> class.</summary>
|
||||
/// <param name="client">The gateway's Galaxy repository client.</param>
|
||||
/// <param name="pageSize">Objects per page (<c>Repository.DiscoverPageSize</c>); non-positive → the client default.</param>
|
||||
public GatewayGalaxyPlatformSource(GalaxyRepositoryClient client, int pageSize = GatewayGalaxyHierarchySource.DefaultPageSize)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
_pageSize = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
=> QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every
|
||||
/// request carries the template filter and <c>include_attributes = false</c>; the reply is
|
||||
/// re-filtered on the returned template chain (a gateway that ignores the filter still
|
||||
/// yields only platforms) and reduced to non-empty tag names.
|
||||
/// </summary>
|
||||
/// <param name="discover">The raw <c>DiscoverHierarchy</c> RPC.</param>
|
||||
/// <param name="pageSize">Objects per page.</param>
|
||||
/// <param name="cancellationToken">Cancellation token for the walk.</param>
|
||||
/// <returns>Distinct platform tag names in gateway order.</returns>
|
||||
internal static async Task<IReadOnlyList<string>> QueryAsync(
|
||||
Func<DiscoverHierarchyRequest, CancellationToken, Task<DiscoverHierarchyReply>> discover,
|
||||
int pageSize,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(discover);
|
||||
var effective = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize);
|
||||
|
||||
var names = new List<string>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
|
||||
var pageToken = string.Empty;
|
||||
do
|
||||
{
|
||||
var request = new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = effective,
|
||||
PageToken = pageToken,
|
||||
IncludeAttributes = false,
|
||||
};
|
||||
request.TemplateChainContains.Add(PlatformTemplates);
|
||||
|
||||
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var obj in reply.Objects)
|
||||
{
|
||||
if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
|
||||
if (seen.Add(obj.TagName)) names.Add(obj.TagName);
|
||||
}
|
||||
|
||||
pageToken = reply.NextPageToken ?? string.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop.");
|
||||
}
|
||||
}
|
||||
while (!string.IsNullOrWhiteSpace(pageToken));
|
||||
|
||||
return names;
|
||||
}
|
||||
|
||||
private static bool IsPlatform(GalaxyObject obj) =>
|
||||
obj.TemplateChain.Count == 0 // a gateway that strips the chain already filtered server-side
|
||||
|| obj.TemplateChain.Any(t => PlatformTemplates.Contains(t, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
/// <summary>
|
||||
/// Driver-side seam for the set of Galaxy platform / engine objects whose <c>ScanState</c>
|
||||
/// the <c>PerPlatformProbeWatcher</c> advises. Production wraps the gateway's
|
||||
/// <c>DiscoverHierarchy</c> RPC filtered to <c>$WinPlatform</c> / <c>$AppEngine</c> with no
|
||||
/// attributes (<see cref="GatewayGalaxyPlatformSource"/>); tests substitute a canned list.
|
||||
/// The driver seeds the watcher from this at <c>InitializeAsync</c> and re-seeds on every
|
||||
/// deploy-watcher rediscovery — the runtime never calls <c>DiscoverAsync</c> on a live driver,
|
||||
/// so without this seed the probe watcher never had a platform set in production.
|
||||
/// </summary>
|
||||
public interface IGalaxyPlatformSource
|
||||
{
|
||||
/// <summary>The <c>tag_name</c> of every deployed <c>$WinPlatform</c> / <c>$AppEngine</c> object.</summary>
|
||||
/// <param name="cancellationToken">Cancellation token for the query.</param>
|
||||
/// <returns>Platform tag names; empty when the galaxy has none.</returns>
|
||||
Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -54,12 +54,18 @@ public sealed class GalaxyDriverProbe : IDriverProbe
|
||||
if (string.IsNullOrWhiteSpace(host) || port <= 0)
|
||||
return new(false, "Config has no host/port to probe.", null);
|
||||
|
||||
// The config's own ProbeTimeoutSeconds caps the probe deadline. It can only tighten the
|
||||
// caller's budget, never extend it — the caller's token fires at `timeout` regardless.
|
||||
timeout = EffectiveTimeout(timeout, opts.ProbeTimeoutSeconds);
|
||||
using var probeCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
probeCts.CancelAfter(timeout);
|
||||
|
||||
// --- TCP preflight: fast-fail for closed ports / unreachable hosts ---
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(host, port, ct);
|
||||
await socket.ConnectAsync(host, port, probeCts.Token);
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
@@ -86,10 +92,7 @@ public sealed class GalaxyDriverProbe : IDriverProbe
|
||||
{
|
||||
client = GalaxyRepositoryClient.Create(BuildProbeClientOptions(opts.Gateway, timeout));
|
||||
|
||||
using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
deadlineCts.CancelAfter(timeout);
|
||||
|
||||
await client.TestConnectionAsync(deadlineCts.Token).ConfigureAwait(false);
|
||||
await client.TestConnectionAsync(probeCts.Token).ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
// No RpcException ⇒ the gateway answered the gRPC call successfully.
|
||||
@@ -157,6 +160,26 @@ public sealed class GalaxyDriverProbe : IDriverProbe
|
||||
_ => (false, $"Reachable at {host}:{port} but gateway gRPC handshake failed: {code}"),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The deadline the probe actually runs under: the caller's budget, tightened by the
|
||||
/// config's <see cref="GalaxyDriverOptions.ProbeTimeoutSeconds"/> when that is positive and
|
||||
/// shorter. The knob was authored by the AdminUI and parsed by the factory but consumed by
|
||||
/// nothing — the probe always ran on the caller's budget alone. A non-positive caller budget
|
||||
/// is floored to 1 s before the comparison (the same floor <see cref="BuildProbeClientOptions"/>
|
||||
/// applies to the gRPC call budget), so the probe runs briefly rather than cancelling before
|
||||
/// it starts; a longer config value never raises it above that floor.
|
||||
/// </summary>
|
||||
/// <param name="callerTimeout">The budget the host handed the probe.</param>
|
||||
/// <param name="configuredSeconds">The config's <c>ProbeTimeoutSeconds</c>; ignored when not positive.</param>
|
||||
/// <returns>The effective probe deadline.</returns>
|
||||
internal static TimeSpan EffectiveTimeout(TimeSpan callerTimeout, int configuredSeconds)
|
||||
{
|
||||
var caller = callerTimeout > TimeSpan.Zero ? callerTimeout : TimeSpan.FromSeconds(1);
|
||||
if (configuredSeconds <= 0) return caller;
|
||||
var configured = TimeSpan.FromSeconds(configuredSeconds);
|
||||
return configured < caller ? configured : caller;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the gRPC client options for the probe ping from the gateway config WITHOUT
|
||||
/// resolving the API-key secret ref (the host owns secret resolution; the probe sends the
|
||||
|
||||
@@ -38,12 +38,19 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
private readonly ILogger _logger;
|
||||
private readonly int _bufferedUpdateIntervalMs;
|
||||
|
||||
// The driver's fan-out registry. The EventPump resolves every inbound event through it, so
|
||||
// the probe handles must be registered there (under SubscriptionRegistry.ProbeSubscriptionId)
|
||||
// or every ScanState event is dropped as "stale after unsubscribe". Null when the watcher is
|
||||
// driven directly (unit tests of the decoding table).
|
||||
private readonly SubscriptionRegistry? _registry;
|
||||
|
||||
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
|
||||
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
|
||||
private readonly ConcurrentDictionary<string, int> _itemHandlesByPlatform =
|
||||
new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Lock _syncLock = new();
|
||||
private bool _disposed;
|
||||
// Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop.
|
||||
private volatile bool _disposed;
|
||||
|
||||
/// <summary>Initializes a new instance of the PerPlatformProbeWatcher class.</summary>
|
||||
/// <param name="subscriber">The Galaxy subscriber for managing probe subscriptions.</param>
|
||||
@@ -55,9 +62,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
HostStatusAggregator aggregator,
|
||||
ILogger? logger = null,
|
||||
int bufferedUpdateIntervalMs = 0)
|
||||
: this(subscriber, aggregator, registry: null, logger, bufferedUpdateIntervalMs)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Driver-side ctor: the watcher publishes its probe handles into <paramref name="registry"/>
|
||||
/// under <see cref="SubscriptionRegistry.ProbeSubscriptionId"/> so the <see cref="EventPump"/>
|
||||
/// fans ScanState events out to <see cref="OnProbeValueChanged"/>.
|
||||
/// </summary>
|
||||
/// <param name="subscriber">The Galaxy subscriber for managing probe subscriptions.</param>
|
||||
/// <param name="aggregator">The host status aggregator for tracking platform connectivity.</param>
|
||||
/// <param name="registry">The driver's subscription registry; null to keep the handles private.</param>
|
||||
/// <param name="logger">Optional logger for diagnostic messages.</param>
|
||||
/// <param name="bufferedUpdateIntervalMs">Buffered update interval in milliseconds; must be >= 0.</param>
|
||||
internal PerPlatformProbeWatcher(
|
||||
IGalaxySubscriber subscriber,
|
||||
HostStatusAggregator aggregator,
|
||||
SubscriptionRegistry? registry,
|
||||
ILogger? logger = null,
|
||||
int bufferedUpdateIntervalMs = 0)
|
||||
{
|
||||
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
|
||||
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
|
||||
_registry = registry;
|
||||
_logger = logger ?? NullLogger.Instance;
|
||||
if (bufferedUpdateIntervalMs < 0)
|
||||
{
|
||||
@@ -67,6 +95,103 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
_bufferedUpdateIntervalMs = bufferedUpdateIntervalMs;
|
||||
}
|
||||
|
||||
/// <summary>Current platform → probe-handle bindings, as the registry sees them. Test seam.</summary>
|
||||
internal IReadOnlyList<TagBinding> CurrentBindings => BuildBindings();
|
||||
|
||||
private IReadOnlyList<TagBinding> BuildBindings() =>
|
||||
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))];
|
||||
|
||||
/// <summary>
|
||||
/// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped —
|
||||
/// a stopped watcher must never write its entry back into a registry the teardown cleared).
|
||||
/// </summary>
|
||||
private void PublishBindings()
|
||||
{
|
||||
lock (_syncLock) PublishBindingsLocked();
|
||||
}
|
||||
|
||||
private void PublishBindingsLocked()
|
||||
{
|
||||
if (_registry is null || _disposed) return;
|
||||
var bindings = BuildBindings();
|
||||
if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId);
|
||||
else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-issue SubscribeBulk for every tracked platform and replace the handles — the
|
||||
/// reconnect replay path, where the gateway session was recreated and every pre-reconnect
|
||||
/// handle is dead. Platforms the gateway now rejects keep a 0 handle (no events expected).
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>The number of probe references re-subscribed.</returns>
|
||||
public async Task<int> ResubscribeAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
var platforms = _itemHandlesByPlatform.Keys.ToArray();
|
||||
if (platforms.Length == 0) return 0;
|
||||
|
||||
await SubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false);
|
||||
return platforms.Length;
|
||||
}
|
||||
|
||||
private async Task SubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
|
||||
{
|
||||
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
|
||||
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
|
||||
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
|
||||
// changes are rare so a coarser interval is usually fine; deployments that need
|
||||
// tighter health visibility can dial it down through GalaxyDriverOptions.
|
||||
var results = await _subscriber.SubscribeBulkAsync(
|
||||
probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var acquired = new (string Platform, int ItemHandle, string? Error)[platforms.Count];
|
||||
for (var i = 0; i < platforms.Count; i++)
|
||||
{
|
||||
var match = results.FirstOrDefault(r => string.Equals(
|
||||
r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
|
||||
acquired[i] = (platforms[i], match is { WasSuccessful: true } ? match.ItemHandle : 0,
|
||||
match?.ErrorMessage ?? "<no result returned>");
|
||||
}
|
||||
|
||||
// Re-check the stop AFTER the await, atomically with the record + publish: a StopAsync that
|
||||
// ran while SubscribeBulk was in flight already cleared the map and the registry entry, so
|
||||
// recording now would resurrect the probe entry into the next runtime's registry.
|
||||
bool stopped;
|
||||
lock (_syncLock)
|
||||
{
|
||||
stopped = _disposed;
|
||||
if (!stopped)
|
||||
{
|
||||
foreach (var (platform, itemHandle, _) in acquired) _itemHandlesByPlatform[platform] = itemHandle;
|
||||
PublishBindingsLocked();
|
||||
}
|
||||
}
|
||||
|
||||
if (stopped)
|
||||
{
|
||||
var orphans = acquired.Where(a => a.ItemHandle > 0).Select(a => a.ItemHandle).ToArray();
|
||||
if (orphans.Length > 0)
|
||||
{
|
||||
try { await _subscriber.UnsubscribeBulkAsync(orphans, CancellationToken.None).ConfigureAwait(false); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (platform, itemHandle, error) in acquired)
|
||||
{
|
||||
if (itemHandle <= 0)
|
||||
{
|
||||
_logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Snapshot of platform tag names currently watched.</summary>
|
||||
public IReadOnlyCollection<string> WatchedPlatforms => [.. _itemHandlesByPlatform.Keys];
|
||||
|
||||
@@ -109,6 +234,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
|
||||
if (toRemove.Count > 0)
|
||||
{
|
||||
// Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe.
|
||||
PublishBindings();
|
||||
try
|
||||
{
|
||||
await _subscriber.UnsubscribeBulkAsync(
|
||||
@@ -124,30 +251,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
|
||||
if (toAdd.Count == 0) return;
|
||||
|
||||
var probeAddresses = toAdd.Select(p => p + ProbeSuffix).ToArray();
|
||||
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
|
||||
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
|
||||
// changes are rare so a coarser interval is usually fine; deployments that need
|
||||
// tighter health visibility can dial it down through GalaxyDriverOptions.
|
||||
var results = await _subscriber.SubscribeBulkAsync(
|
||||
probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
for (var i = 0; i < toAdd.Count; i++)
|
||||
{
|
||||
var platform = toAdd[i];
|
||||
var match = results.FirstOrDefault(r => string.Equals(
|
||||
r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0;
|
||||
_itemHandlesByPlatform[platform] = itemHandle;
|
||||
|
||||
if (itemHandle <= 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}",
|
||||
platform, match?.ErrorMessage ?? "<no result returned>");
|
||||
}
|
||||
}
|
||||
await SubscribeAndRecordAsync(toAdd, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -196,20 +300,32 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Disposes the probe watcher and unsubscribes all tracked platforms.</summary>
|
||||
public void Dispose()
|
||||
/// <summary>
|
||||
/// Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway
|
||||
/// (best-effort — a transport error is logged and swallowed). The async counterpart of
|
||||
/// <see cref="Dispose"/>; the driver's <c>ShutdownAsync</c> uses it so the unsubscribe
|
||||
/// round-trip is awaited rather than blocked on. Idempotent.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Bounds the unsubscribe round-trip.</param>
|
||||
/// <returns>A task that completes once the handles have been released (or the attempt gave up).</returns>
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
|
||||
// Best-effort unsubscribe everything we know about. Run synchronously through
|
||||
// GetAwaiter().GetResult() since Dispose is sync; transport errors are swallowed.
|
||||
var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray();
|
||||
_itemHandlesByPlatform.Clear();
|
||||
int[] liveHandles;
|
||||
lock (_syncLock)
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
liveHandles = [.. _itemHandlesByPlatform.Values.Where(h => h > 0)];
|
||||
_itemHandlesByPlatform.Clear();
|
||||
_registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId);
|
||||
}
|
||||
if (liveHandles.Length > 0)
|
||||
{
|
||||
try { _subscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher dispose unsubscribe failed"); }
|
||||
try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip).</summary>
|
||||
public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
@@ -187,9 +187,11 @@ internal sealed class EventPump : IAsyncDisposable
|
||||
NotifyStreamFault(cause);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
catch (Exception) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Clean shutdown — no log.
|
||||
// Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled)
|
||||
// rather than OperationCanceledException; either way the pump was told to stop, so this
|
||||
// is not a transport fault and must not wake the reconnect supervisor mid-teardown.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -283,14 +285,19 @@ internal sealed class EventPump : IAsyncDisposable
|
||||
EventsDispatched.Add(1, _clientTag);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
catch (Exception) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Clean shutdown.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// The read loop is still draining the stream into a channel nobody reads: the pump is
|
||||
// as dead as after a transport fault, but used to raise nothing — health stayed
|
||||
// Healthy and the comms-loss watchdog never tripped (a clean termination evading the
|
||||
// watchdog). Report it as a stream fault so the supervisor recreates the pump.
|
||||
_logger.LogWarning(ex,
|
||||
"Galaxy EventPump dispatch loop ended with an exception — events past this point will be lost until restart.");
|
||||
"Galaxy EventPump dispatch loop ended with an exception — notifying reconnect supervisor so the pump is recreated.");
|
||||
NotifyStreamFault(ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -60,6 +60,12 @@ public sealed class GalaxyMxSession : IAsyncDisposable
|
||||
/// </summary>
|
||||
internal Func<CancellationToken, Task>? OpenAndRegisterOverrideForTests { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Test seam — replaces the gateway <c>CloseSession</c> round-trip in <see cref="CloseAsync"/>
|
||||
/// so the bounded-close behaviour can be exercised against a never-answering fake.
|
||||
/// </summary>
|
||||
internal Func<CancellationToken, Task>? CloseOverrideForTests { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Connect the underlying gateway client + open an MXAccess session + register the
|
||||
/// configured client name. Idempotent — second calls are no-ops while a connect has
|
||||
@@ -96,7 +102,7 @@ public sealed class GalaxyMxSession : IAsyncDisposable
|
||||
{
|
||||
// A partial open (e.g. OpenSession succeeded but Register threw) must not leave a half-open
|
||||
// handle that blocks the next reconnect. Tear the partial state down so a retry rebuilds cleanly.
|
||||
await TeardownSessionAsync().ConfigureAwait(false);
|
||||
await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -112,10 +118,23 @@ public sealed class GalaxyMxSession : IAsyncDisposable
|
||||
public async Task RecreateAsync(MxGatewayClientOptions clientOptions, CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
await TeardownSessionAsync().ConfigureAwait(false);
|
||||
await TeardownSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Close the gateway session and release the owned client under
|
||||
/// <paramref name="cancellationToken"/>. The SDK's <c>DisposeAsync</c> closes with NO token,
|
||||
/// i.e. under the 30 s <c>DefaultCallTimeout</c> — against a blackholed gateway that parks
|
||||
/// the driver's <c>ShutdownAsync</c> (and the host's <c>PostStop</c>) for the full timeout.
|
||||
/// A close that is cancelled or fails is NOT retried through the SDK dispose (which would
|
||||
/// park again); the channel is dropped and the gateway reaps the session on its idle lease.
|
||||
/// The session can be re-opened afterwards (this does not set <c>_disposed</c>).
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Bounds the gateway <c>CloseSession</c> round-trip.</param>
|
||||
/// <returns>A task that completes once the session and client are released.</returns>
|
||||
public Task CloseAsync(CancellationToken cancellationToken) => TeardownSessionAsync(cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Test seam — attach a session opened externally (e.g. against an in-process gw
|
||||
/// fake). Skips the gateway-client construction so tests can drive the session
|
||||
@@ -143,24 +162,45 @@ public sealed class GalaxyMxSession : IAsyncDisposable
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
await TeardownSessionAsync().ConfigureAwait(false);
|
||||
await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resets the connect flag and best-effort disposes + nulls the owned session and client.
|
||||
/// Shared by <see cref="DisposeAsync"/> (final teardown) and <see cref="RecreateAsync"/> /
|
||||
/// the partial-open recovery path (rebuild teardown) — it does NOT set <c>_disposed</c>, so
|
||||
/// a torn-down session can be re-opened.
|
||||
/// Resets the connect flag and best-effort closes + nulls the owned session and client.
|
||||
/// Shared by <see cref="DisposeAsync"/> (final teardown), <see cref="CloseAsync"/> (bounded
|
||||
/// teardown) and <see cref="RecreateAsync"/> / the partial-open recovery path (rebuild
|
||||
/// teardown) — it does NOT set <c>_disposed</c>, so a torn-down session can be re-opened.
|
||||
/// </summary>
|
||||
private async Task TeardownSessionAsync()
|
||||
private async Task TeardownSessionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var wasConnected = _connected;
|
||||
_connected = false;
|
||||
|
||||
// The override models the gateway round-trip, so it runs only when there was a live
|
||||
// session to close — a second teardown (CloseAsync then DisposeAsync) must not close twice.
|
||||
var closeOverride = CloseOverrideForTests;
|
||||
if (closeOverride is not null && wasConnected)
|
||||
{
|
||||
try { await closeOverride(cancellationToken).ConfigureAwait(false); }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession close failed or timed out (best-effort)"); }
|
||||
}
|
||||
|
||||
if (_session is not null)
|
||||
{
|
||||
try { await _session.DisposeAsync().ConfigureAwait(false); }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession session dispose failed (best-effort)"); }
|
||||
var session = _session;
|
||||
_session = null;
|
||||
try
|
||||
{
|
||||
// CloseAsync(ct) is the bounded gateway round-trip; DisposeAsync only re-runs an
|
||||
// un-replied close (with no token), so it is reached only after a successful one.
|
||||
await session.CloseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await session.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"GalaxyMxSession close failed or timed out (best-effort) — dropping the channel; the gateway reaps the session on its idle lease.");
|
||||
}
|
||||
}
|
||||
|
||||
if (_ownedClient is not null)
|
||||
|
||||
@@ -120,15 +120,87 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
|
||||
// Bind the per-batch MX write operations behind the internal IMxWriteOps seam (see below) so the
|
||||
// whole write pipeline is unit-testable without the sealed SDK session (archreview 06/S-1).
|
||||
var ops = new SessionMxWriteOps(session, serverHandle);
|
||||
return await WriteBatchAsync(ops, writes, securityResolver, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Upper bound on write lanes in flight at once. A first-touch write is three round-trips
|
||||
/// (AddItem → AdviseSupervisory → Write) and the batch used to run them strictly serially;
|
||||
/// distinct tags are independent, so their lanes now overlap, bounded so a large batch cannot
|
||||
/// flood the gateway's STA pump. Acks stay serial by design (see <c>GalaxyDriver.AcknowledgeAsync</c>).
|
||||
/// </summary>
|
||||
internal const int MaxConcurrentWriteLanes = 8;
|
||||
|
||||
/// <summary>Test seam: drive the real batch pipeline against a fake <see cref="IMxWriteOps"/>.</summary>
|
||||
/// <param name="ops">The (fake) MX write operations.</param>
|
||||
/// <param name="writes">The batch.</param>
|
||||
/// <param name="securityResolver">Per-tag classification resolver.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>One result per request, in input order.</returns>
|
||||
internal Task<IReadOnlyList<WriteResult>> WriteBatchForTestAsync(
|
||||
IMxWriteOps ops, IReadOnlyList<WriteRequest> writes,
|
||||
Func<string, SecurityClassification> securityResolver, CancellationToken ct)
|
||||
=> WriteBatchAsync(ops, writes, securityResolver, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Run a batch as one lane per distinct tag (OrdinalIgnoreCase): writes to the SAME tag
|
||||
/// stay in input order inside their lane, lanes run concurrently up to
|
||||
/// <see cref="MaxConcurrentWriteLanes"/>, and results land at their request's index.
|
||||
/// A single-tag batch takes the direct path with no gate.
|
||||
/// </summary>
|
||||
private async Task<IReadOnlyList<WriteResult>> WriteBatchAsync(
|
||||
IMxWriteOps ops, IReadOnlyList<WriteRequest> writes,
|
||||
Func<string, SecurityClassification> securityResolver, CancellationToken ct)
|
||||
{
|
||||
var results = new WriteResult[writes.Count];
|
||||
if (writes.Count == 0) return results;
|
||||
|
||||
// Group request indices by tag, first-seen order preserved so the lane start order is
|
||||
// deterministic and a same-tag sequence keeps its authored order.
|
||||
var lanes = new Dictionary<string, List<int>>(StringComparer.OrdinalIgnoreCase);
|
||||
var laneOrder = new List<List<int>>();
|
||||
for (var i = 0; i < writes.Count; i++)
|
||||
{
|
||||
results[i] = await WriteOneAsync(ops, writes[i],
|
||||
securityResolver(writes[i].FullReference), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (!lanes.TryGetValue(writes[i].FullReference, out var lane))
|
||||
{
|
||||
lane = [];
|
||||
lanes[writes[i].FullReference] = lane;
|
||||
laneOrder.Add(lane);
|
||||
}
|
||||
lane.Add(i);
|
||||
}
|
||||
|
||||
if (laneOrder.Count == 1)
|
||||
{
|
||||
await RunLaneAsync(laneOrder[0]).ConfigureAwait(false);
|
||||
return results;
|
||||
}
|
||||
|
||||
using var gate = new SemaphoreSlim(MaxConcurrentWriteLanes, MaxConcurrentWriteLanes);
|
||||
var tasks = new Task[laneOrder.Count];
|
||||
for (var l = 0; l < laneOrder.Count; l++)
|
||||
{
|
||||
var lane = laneOrder[l];
|
||||
tasks[l] = RunGatedAsync(lane);
|
||||
}
|
||||
await Task.WhenAll(tasks).ConfigureAwait(false);
|
||||
return results;
|
||||
|
||||
async Task RunGatedAsync(List<int> lane)
|
||||
{
|
||||
await gate.WaitAsync(ct).ConfigureAwait(false);
|
||||
try { await RunLaneAsync(lane).ConfigureAwait(false); }
|
||||
finally { gate.Release(); }
|
||||
}
|
||||
|
||||
async Task RunLaneAsync(List<int> lane)
|
||||
{
|
||||
foreach (var i in lane)
|
||||
{
|
||||
results[i] = await WriteOneAsync(ops, writes[i],
|
||||
securityResolver(writes[i].FullReference), ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test seam: drive the real single-write pipeline against a fake <see cref="IMxWriteOps"/>
|
||||
|
||||
@@ -62,6 +62,16 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
private Task? _recoveryLoop;
|
||||
private CancellationTokenSource? _loopCts;
|
||||
private bool _disposed;
|
||||
private int _cyclesStarted;
|
||||
private int _cyclesReleased;
|
||||
|
||||
/// <summary>Recovery cycles started so far (one per <see cref="ReportTransportFailure"/> that left Healthy). Test seam.</summary>
|
||||
internal int CyclesStarted => Volatile.Read(ref _cyclesStarted);
|
||||
|
||||
/// <summary>Recovery cycles whose <see cref="CancellationTokenSource"/> has been disposed. Equals
|
||||
/// <see cref="CyclesStarted"/> once every loop has exited — a cycle's CTS used to be released only
|
||||
/// if it happened to be the LAST one when <see cref="Dispose"/> ran. Test seam.</summary>
|
||||
internal int CyclesReleased => Volatile.Read(ref _cyclesReleased);
|
||||
|
||||
/// <summary>Fires after every state transition.</summary>
|
||||
public event EventHandler<StateTransition>? StateChanged;
|
||||
@@ -136,8 +146,13 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
|
||||
TransitionLocked(State.TransportLost, cause.Message);
|
||||
|
||||
_loopCts = new CancellationTokenSource();
|
||||
_recoveryLoop = Task.Run(() => RecoveryLoopAsync(_loopCts.Token));
|
||||
// Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's
|
||||
// finally). Before, every cycle allocated a fresh CTS and only the last one was ever
|
||||
// disposed — one leaked CancellationTokenSource per outage for the driver's lifetime.
|
||||
var cts = new CancellationTokenSource();
|
||||
_loopCts = cts;
|
||||
Interlocked.Increment(ref _cyclesStarted);
|
||||
_recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,7 +171,26 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoveryLoopAsync(CancellationToken ct)
|
||||
private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may
|
||||
// race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object.
|
||||
lock (_stateLock)
|
||||
{
|
||||
if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null;
|
||||
}
|
||||
cycleCts.Dispose();
|
||||
Interlocked.Increment(ref _cyclesReleased);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoveryLoopCoreAsync(CancellationToken ct)
|
||||
{
|
||||
var attempt = 0;
|
||||
while (!ct.IsCancellationRequested)
|
||||
@@ -240,8 +274,13 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Disposes the supervisor and cancels any in-flight recovery loop.</summary>
|
||||
public void Dispose()
|
||||
/// <summary>
|
||||
/// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's
|
||||
/// thread. The async counterpart of <see cref="Dispose"/>; the driver's <c>ShutdownAsync</c>
|
||||
/// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent.
|
||||
/// </summary>
|
||||
/// <returns>A task that completes once the recovery loop has exited.</returns>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
@@ -250,13 +289,18 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
Task? loop;
|
||||
lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }
|
||||
|
||||
cts?.Cancel();
|
||||
// The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
|
||||
// throws on Cancel, and a disposed one is a no-op on Dispose.
|
||||
try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ }
|
||||
if (loop is not null)
|
||||
{
|
||||
try { loop.GetAwaiter().GetResult(); } catch { /* shutdown */ }
|
||||
try { await loop.ConfigureAwait(false); } catch { /* shutdown */ }
|
||||
}
|
||||
cts?.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits).</summary>
|
||||
public void Dispose() => StopAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -18,10 +18,39 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
/// </remarks>
|
||||
internal sealed class SubscriptionRegistry
|
||||
{
|
||||
/// <summary>
|
||||
/// Reserved subscription id for the per-platform <c>ScanState</c> probe handles
|
||||
/// (<c>PerPlatformProbeWatcher</c>). Driver subscription ids are positive and
|
||||
/// monotonic (<see cref="NextSubscriptionId"/>), so a negative id can never collide.
|
||||
/// The probe handles used to live only in the watcher's own map, so the
|
||||
/// <see cref="EventPump"/> — which resolves subscribers through THIS registry —
|
||||
/// dropped every probe event as "stale after unsubscribe" and per-platform host
|
||||
/// status never updated.
|
||||
/// </summary>
|
||||
public const long ProbeSubscriptionId = -1;
|
||||
|
||||
private readonly ConcurrentDictionary<long, SubscriptionEntry> _bySubscriptionId = new();
|
||||
private readonly ConcurrentDictionary<int, ImmutableHashSet<long>> _subscribersByItemHandle = new();
|
||||
private long _nextSubscriptionId;
|
||||
|
||||
/// <summary>True when <paramref name="subscriptionId"/> is currently registered.</summary>
|
||||
/// <param name="subscriptionId">The subscription identifier.</param>
|
||||
/// <returns>Whether the id is tracked.</returns>
|
||||
public bool Contains(long subscriptionId) => _bySubscriptionId.ContainsKey(subscriptionId);
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="Rebind"/> when the id is already tracked (so its stale reverse-map entries
|
||||
/// are dropped), <see cref="Register"/> otherwise. Used by the probe watcher, whose single
|
||||
/// reserved subscription is re-issued on every platform-set change.
|
||||
/// </summary>
|
||||
/// <param name="subscriptionId">The subscription identifier.</param>
|
||||
/// <param name="bindings">The tag bindings for the subscription.</param>
|
||||
public void Upsert(long subscriptionId, IReadOnlyList<TagBinding> bindings)
|
||||
{
|
||||
if (Contains(subscriptionId)) Rebind(subscriptionId, bindings);
|
||||
else Register(subscriptionId, bindings);
|
||||
}
|
||||
|
||||
/// <summary>Gets the number of tracked subscriptions.</summary>
|
||||
public int TrackedSubscriptionCount => _bySubscriptionId.Count;
|
||||
/// <summary>Gets the number of tracked item handles.</summary>
|
||||
@@ -100,6 +129,17 @@ internal sealed class SubscriptionRegistry
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Drop every subscription and the whole reverse map. Used by the driver's runtime teardown:
|
||||
/// once the gateway session is closed every item handle in here is dead, and the host
|
||||
/// re-issues its subscriptions (fresh ids) after the next <c>InitializeAsync</c>.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
_bySubscriptionId.Clear();
|
||||
_subscribersByItemHandle.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Snapshot every active binding for diagnostic output.</summary>
|
||||
/// <returns>A flattened list of every tag binding across all tracked subscriptions.</returns>
|
||||
public IReadOnlyList<TagBinding> SnapshotAllBindings() =>
|
||||
|
||||
+2
-2
@@ -92,7 +92,7 @@
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Discover page size</label>
|
||||
<InputNumber @bind-Value="_form.Galaxy.RepositoryDiscoverPageSize" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Default 5000 objects per page.</div>
|
||||
<div class="form-text">Objects per DiscoverHierarchy page the driver asks the gateway for. Default 5000.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="form-check form-switch mt-4">
|
||||
@@ -138,7 +138,7 @@
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Admin UI probe timeout (seconds)</label>
|
||||
<InputNumber @bind-Value="_form.Galaxy.ProbeTimeoutSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Max 60. Used by Test Connect. Default 30.</div>
|
||||
<div class="form-text">Caps the Test Connect probe deadline (the probe's own budget still applies, so this only shortens it). Max 60. Default 30.</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-5: <c>Repository.DiscoverPageSize</c> was authored and parsed but reached
|
||||
/// nothing — the client's typed <c>DiscoverHierarchyAsync</c> hard-codes 5000. The source now pages
|
||||
/// the raw RPC itself at the configured size.
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyHierarchySourcePagingTests
|
||||
{
|
||||
private static GalaxyObject Obj(int id) => new() { GobjectId = id, TagName = $"Obj{id}" };
|
||||
|
||||
/// <summary>Every request carries the configured page size and the page tokens chain until the last page.</summary>
|
||||
[Fact]
|
||||
public async Task Pages_at_the_configured_size_and_chains_tokens()
|
||||
{
|
||||
var requests = new List<DiscoverHierarchyRequest>();
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
requests.Add(req);
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
switch (req.PageToken)
|
||||
{
|
||||
case "":
|
||||
reply.Objects.Add([Obj(1), Obj(2)]);
|
||||
reply.NextPageToken = "p2";
|
||||
break;
|
||||
case "p2":
|
||||
reply.Objects.Add([Obj(3), Obj(4)]);
|
||||
reply.NextPageToken = "p3";
|
||||
break;
|
||||
default:
|
||||
reply.Objects.Add(Obj(5));
|
||||
reply.NextPageToken = "";
|
||||
break;
|
||||
}
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var objects = await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 2, CancellationToken.None);
|
||||
|
||||
objects.Select(o => o.GobjectId).ShouldBe([1, 2, 3, 4, 5]);
|
||||
requests.Count.ShouldBe(3);
|
||||
requests.ShouldAllBe(r => r.PageSize == 2);
|
||||
requests.Select(r => r.PageToken).ShouldBe(["", "p2", "p3"]);
|
||||
}
|
||||
|
||||
/// <summary>A different configured size is what goes on the wire (positive control for the knob).</summary>
|
||||
[Fact]
|
||||
public async Task A_different_page_size_is_sent_on_the_wire()
|
||||
{
|
||||
int? sent = null;
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
sent = req.PageSize;
|
||||
return Task.FromResult(new DiscoverHierarchyReply());
|
||||
}
|
||||
|
||||
await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 250, CancellationToken.None);
|
||||
sent.ShouldBe(250);
|
||||
}
|
||||
|
||||
/// <summary>A non-positive configured size falls back to the client library's 5000 default.</summary>
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-7)]
|
||||
public void Non_positive_page_size_falls_back_to_the_default(int configured)
|
||||
=> GatewayGalaxyHierarchySource.EffectivePageSize(configured).ShouldBe(GatewayGalaxyHierarchySource.DefaultPageSize);
|
||||
|
||||
/// <summary>A repeated page token is refused rather than walked forever.</summary>
|
||||
[Fact]
|
||||
public async Task A_repeated_page_token_is_refused()
|
||||
{
|
||||
var calls = 0;
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
calls++;
|
||||
var reply = new DiscoverHierarchyReply { NextPageToken = "same" };
|
||||
reply.Objects.Add(Obj(calls));
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 10, CancellationToken.None));
|
||||
ex.Message.ShouldContain("repeated page token");
|
||||
calls.ShouldBe(2);
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
|
||||
|
||||
/// <summary>The platform-only query the driver seeds its ScanState probe watcher from.</summary>
|
||||
public sealed class GatewayGalaxyPlatformSourceTests
|
||||
{
|
||||
private static GalaxyObject Obj(string tag, params string[] chain)
|
||||
{
|
||||
var o = new GalaxyObject { TagName = tag };
|
||||
o.TemplateChain.Add(chain);
|
||||
return o;
|
||||
}
|
||||
|
||||
/// <summary>Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered.</summary>
|
||||
[Fact]
|
||||
public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered()
|
||||
{
|
||||
var requests = new List<DiscoverHierarchyRequest>();
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
requests.Add(req);
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
if (req.PageToken == "")
|
||||
{
|
||||
reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]);
|
||||
reply.NextPageToken = "p2";
|
||||
}
|
||||
else
|
||||
{
|
||||
reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]);
|
||||
}
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None);
|
||||
|
||||
names.ShouldBe(["Plat01", "Eng01"]);
|
||||
requests.Count.ShouldBe(2);
|
||||
requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false);
|
||||
requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine"));
|
||||
}
|
||||
|
||||
/// <summary>A gateway that strips the template chain is trusted to have filtered server-side.</summary>
|
||||
[Fact]
|
||||
public async Task Objects_without_a_template_chain_are_kept()
|
||||
{
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
reply.Objects.Add(new GalaxyObject { TagName = "Plat01" });
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
(await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 0, CancellationToken.None)).ShouldBe(["Plat01"]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-6: <c>ProbeTimeoutSeconds</c> was authored and compared for equivalence but
|
||||
/// never consumed — the probe ran on the caller's budget alone. It now caps the probe deadline
|
||||
/// (it can shorten the caller's budget, never extend it).
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class GalaxyDriverProbeTimeoutTests
|
||||
{
|
||||
/// <summary>The effective deadline is the shorter of the caller's budget and the configured cap.</summary>
|
||||
[Theory]
|
||||
[InlineData(10, 30, 10)] // caller shorter → caller wins
|
||||
[InlineData(10, 2, 2)] // config shorter → config caps
|
||||
[InlineData(10, 0, 10)] // unset config → caller
|
||||
[InlineData(10, -1, 10)] // nonsense config → caller
|
||||
[InlineData(0, 5, 1)] // no caller budget → 1 s floor beats a longer config
|
||||
[InlineData(0, 0, 1)] // nothing → 1 s floor
|
||||
public void EffectiveTimeout_is_the_shorter_of_caller_and_config(int callerSeconds, int configSeconds, int expectedSeconds)
|
||||
=> GalaxyDriverProbe.EffectiveTimeout(TimeSpan.FromSeconds(callerSeconds), configSeconds)
|
||||
.ShouldBe(TimeSpan.FromSeconds(expectedSeconds));
|
||||
|
||||
/// <summary>
|
||||
/// Against a port that accepts TCP but never speaks gRPC, a 1 s configured cap returns long
|
||||
/// before the caller's 20 s budget. The gRPC handshake itself does the waiting here, so this
|
||||
/// exercises the real deadline path, not the pure helper.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Configured_cap_shortens_the_probe_against_a_silent_listener()
|
||||
{
|
||||
var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
try
|
||||
{
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
var config = $$"""
|
||||
{
|
||||
"Gateway": { "Endpoint": "http://127.0.0.1:{{port}}", "ApiKeySecretRef": "dev:k", "UseTls": false },
|
||||
"MxAccess": { "ClientName": "probe-timeout" },
|
||||
"Repository": {},
|
||||
"Reconnect": {},
|
||||
"ProbeTimeoutSeconds": 1
|
||||
}
|
||||
""";
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = await new GalaxyDriverProbe().ProbeAsync(config, TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
|
||||
sw.Stop();
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), $"the 1 s configured cap must bound the probe (message: {result.Message})");
|
||||
result.Message.ShouldNotBeNull();
|
||||
(result.Message!.Contains("timed out", StringComparison.OrdinalIgnoreCase)
|
||||
|| result.Message.Contains("DeadlineExceeded", StringComparison.OrdinalIgnoreCase)
|
||||
|| result.Message.Contains("handshake failed", StringComparison.OrdinalIgnoreCase))
|
||||
.ShouldBeTrue($"unexpected message: '{result.Message}'");
|
||||
}
|
||||
finally
|
||||
{
|
||||
listener.Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-4: the watcher's probe handles must live in the driver's
|
||||
/// <see cref="SubscriptionRegistry"/> under the reserved id, follow the platform set, be
|
||||
/// rebound on <see cref="PerPlatformProbeWatcher.ResubscribeAsync"/>, and leave on stop.
|
||||
/// </summary>
|
||||
public sealed class PerPlatformProbeWatcherRegistryTests
|
||||
{
|
||||
private sealed class FakeSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private int _next;
|
||||
public List<int[]> Unsubscribes { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<SubscribeResult>>(
|
||||
[.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = ++_next, WasSuccessful = true })]);
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
Unsubscribes.Add([.. itemHandles]);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, cancellationToken);
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
|
||||
private static (long Id, string Ref)[] Resolve(SubscriptionRegistry registry, int handle) =>
|
||||
[.. registry.ResolveSubscribers(handle)];
|
||||
|
||||
/// <summary>Synced platforms resolve through the registry under the reserved id; removed ones stop resolving.</summary>
|
||||
[Fact]
|
||||
public async Task Sync_registers_probe_handles_under_the_reserved_id_and_tracks_removals()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
|
||||
|
||||
await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
|
||||
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue();
|
||||
var bindings = watcher.CurrentBindings;
|
||||
bindings.Count.ShouldBe(2);
|
||||
foreach (var b in bindings)
|
||||
{
|
||||
Resolve(registry, b.ItemHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, b.FullReference)]);
|
||||
}
|
||||
|
||||
var bHandle = bindings.Single(b => b.FullReference == "B.ScanState").ItemHandle;
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
Resolve(registry, bHandle).ShouldBeEmpty("a removed platform's handle must leave the fan-out map");
|
||||
var aHandle = bindings.Single(b => b.FullReference == "A.ScanState").ItemHandle;
|
||||
Resolve(registry, aHandle).ShouldHaveSingleItem("positive control: the kept platform still resolves");
|
||||
}
|
||||
|
||||
/// <summary>Resubscribe replaces the handles and rebinds: old handles stop resolving, new ones resolve.</summary>
|
||||
[Fact]
|
||||
public async Task Resubscribe_rebinds_the_registry_to_the_fresh_handles()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
var oldHandle = watcher.CurrentBindings.Single().ItemHandle;
|
||||
|
||||
(await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
|
||||
|
||||
var newHandle = watcher.CurrentBindings.Single().ItemHandle;
|
||||
newHandle.ShouldNotBe(oldHandle);
|
||||
Resolve(registry, oldHandle).ShouldBeEmpty();
|
||||
Resolve(registry, newHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, "A.ScanState")]);
|
||||
registry.TrackedSubscriptionCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>StopAsync unsubscribes the live handles and removes the reserved entry.</summary>
|
||||
[Fact]
|
||||
public async Task StopAsync_unsubscribes_and_removes_the_reserved_entry()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var subscriber = new FakeSubscriber();
|
||||
var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
|
||||
var handles = watcher.CurrentBindings.Select(b => b.ItemHandle).OrderBy(h => h).ToArray();
|
||||
|
||||
await watcher.StopAsync(CancellationToken.None);
|
||||
|
||||
subscriber.Unsubscribes.ShouldHaveSingleItem().OrderBy(h => h).ShouldBe(handles);
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse();
|
||||
registry.TrackedItemHandleCount.ShouldBe(0);
|
||||
await Should.NotThrowAsync(() => watcher.StopAsync(CancellationToken.None)); // idempotent
|
||||
}
|
||||
|
||||
/// <summary>A watcher without a registry (the decoding-table unit tests) still works and registers nothing.</summary>
|
||||
[Fact]
|
||||
public async Task Without_a_registry_the_watcher_keeps_its_handles_private()
|
||||
{
|
||||
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator());
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
watcher.CurrentBindings.ShouldHaveSingleItem();
|
||||
(await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Classes that assert on process-wide telemetry (an <c>ActivityListener</c> on the Galaxy
|
||||
/// ActivitySource, a <c>MeterListener</c> on the Galaxy meter) run with parallelization disabled:
|
||||
/// those listeners see every Galaxy span / measurement in the process, so any parallel class that
|
||||
/// drives a traced subscriber or the data writer — including a driver's background probe seed that
|
||||
/// outlives its test — pollutes their counts (F-FU6, observed flaky in lane F fix round 2).
|
||||
/// </summary>
|
||||
[CollectionDefinition(Name, DisableParallelization = true)]
|
||||
public sealed class ProcessWideTelemetryCollection
|
||||
{
|
||||
/// <summary>The collection name.</summary>
|
||||
public const string Name = "Process-wide Galaxy telemetry";
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-10: a dispatch-loop exit left the read loop draining into a channel
|
||||
/// nobody read, with health Healthy and nothing telling the supervisor — a clean termination
|
||||
/// evading the watchdog. The exit now reports a stream fault. The fault is injected through
|
||||
/// the pump's handle factory, the one dispatch-side seam that runs outside the per-handler
|
||||
/// isolation.
|
||||
/// </summary>
|
||||
public sealed class EventPumpDispatchFaultTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private sealed class ChannelSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<SubscribeResult>>([]);
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitAsync(int handle) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = 1 },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>A dispatch-loop exception reports a stream fault carrying that exception.</summary>
|
||||
[Fact]
|
||||
public async Task Dispatch_loop_exit_reports_a_stream_fault()
|
||||
{
|
||||
var subscriber = new ChannelSubscriber();
|
||||
var registry = new SubscriptionRegistry();
|
||||
registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
|
||||
var fault = new TaskCompletionSource<Exception>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
await using var pump = new EventPump(
|
||||
subscriber, registry,
|
||||
handleFactory: _ => throw new InvalidDataException("synthetic dispatch failure"),
|
||||
channelCapacity: 4, clientName: "DispatchFault",
|
||||
onStreamFault: ex => fault.TrySetResult(ex));
|
||||
pump.Start();
|
||||
|
||||
await subscriber.EmitAsync(7);
|
||||
|
||||
(await Task.WhenAny(fault.Task, Task.Delay(WaitMs))).ShouldBe(fault.Task,
|
||||
"a dispatch-loop exit must be reported as a stream fault");
|
||||
(await fault.Task).ShouldBeOfType<InvalidDataException>();
|
||||
}
|
||||
|
||||
/// <summary>Positive control: the same event with a working handle factory dispatches and reports no fault.</summary>
|
||||
[Fact]
|
||||
public async Task Same_event_with_a_working_factory_dispatches_without_a_fault()
|
||||
{
|
||||
var subscriber = new ChannelSubscriber();
|
||||
var registry = new SubscriptionRegistry();
|
||||
registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
|
||||
var faulted = false;
|
||||
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
await using var pump = new EventPump(
|
||||
subscriber, registry, channelCapacity: 4, clientName: "DispatchFault",
|
||||
onStreamFault: _ => faulted = true);
|
||||
pump.OnDataChange += (_, e) => delivered.TrySetResult(e);
|
||||
pump.Start();
|
||||
|
||||
await subscriber.EmitAsync(7);
|
||||
|
||||
(await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task);
|
||||
(await delivered.Task).FullReference.ShouldBe("Tank01.Level");
|
||||
faulted.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
+406
@@ -0,0 +1,406 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Lane F fix round 1 (review of #735): the probe watcher is seeded at <c>InitializeAsync</c>
|
||||
/// and re-seeded on rediscovery (the runtime never calls <c>DiscoverAsync</c> on a live driver);
|
||||
/// a <c>SubscribeAsync</c> landing mid-teardown cannot create a pump on the dying subscriber; a
|
||||
/// gateway close that never answers cannot park <c>ShutdownAsync</c>; a re-subscribe that
|
||||
/// straddles a teardown never rebinds into the next runtime's registry.
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverFixRound1Tests
|
||||
{
|
||||
internal const int WaitMs = 10_000;
|
||||
|
||||
internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("FixRound1"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
// ───────────────────────────── fakes ─────────────────────────────
|
||||
|
||||
private sealed class ScriptedPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public Func<int, IReadOnlyList<string>> Script { get; init; } = _ => ["Plat01"];
|
||||
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
=> Task.FromResult(Script(Interlocked.Increment(ref _calls)));
|
||||
}
|
||||
|
||||
internal sealed class ParkableSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle;
|
||||
private int _subscribeCalls;
|
||||
|
||||
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public List<string> Subscribed { get; } = [];
|
||||
public int SubscribeCalls => Volatile.Read(ref _subscribeCalls);
|
||||
|
||||
/// <summary>When set, UnsubscribeBulk parks until <see cref="ReleaseUnsubscribe"/>.</summary>
|
||||
public bool ParkUnsubscribe { get; set; }
|
||||
public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult();
|
||||
|
||||
/// <summary>When > 0, that SubscribeBulk call (1-based) parks until <see cref="ReleaseSubscribe"/>.</summary>
|
||||
public int ParkSubscribeCall { get; set; }
|
||||
public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult();
|
||||
|
||||
public async Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var call = Interlocked.Increment(ref _subscribeCalls);
|
||||
if (call == ParkSubscribeCall)
|
||||
{
|
||||
SubscribeEntered.TrySetResult();
|
||||
await _releaseSubscribe.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
var results = new List<SubscribeResult>();
|
||||
lock (Subscribed)
|
||||
{
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = Interlocked.Increment(ref _nextHandle);
|
||||
Subscribed.Add(r);
|
||||
HandleByRef[r] = h;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>Every handle passed to UnsubscribeBulk, in call order.</summary>
|
||||
public List<int> Unsubscribed { get; } = [];
|
||||
|
||||
public async Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
|
||||
if (ParkUnsubscribe)
|
||||
{
|
||||
UnsubscribeEntered.TrySetResult();
|
||||
await _releaseUnsubscribe.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { BoolValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource
|
||||
{
|
||||
public Channel<DeployEvent> Events { get; } = Channel.CreateUnbounded<DeployEvent>();
|
||||
public int Opened;
|
||||
|
||||
public async IAsyncEnumerable<DeployEvent> WatchAsync(
|
||||
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref Opened);
|
||||
await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent
|
||||
{
|
||||
Sequence = sequence,
|
||||
TimeOfLastDeployPresent = true,
|
||||
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
internal static HostState? StateOf(GalaxyDriver driver, string host) =>
|
||||
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
|
||||
|
||||
// ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ─────────────────────────────
|
||||
|
||||
/// <summary>A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses.</summary>
|
||||
[Fact]
|
||||
public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new ScriptedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
// The seed runs in the background (fix round 2, N1) — init does not wait on it.
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the probe watcher must be seeded after init");
|
||||
platforms.Calls.ShouldBe(1);
|
||||
lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
|
||||
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
|
||||
"a ScanState event must surface the seeded platform on GetHostStatuses");
|
||||
}
|
||||
|
||||
/// <summary>A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds.</summary>
|
||||
[Fact]
|
||||
public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var deploys = new ChannelDeployWatchSource();
|
||||
var platforms = new ScriptedPlatformSource
|
||||
{
|
||||
Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"],
|
||||
};
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
|
||||
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
|
||||
var raised = 0;
|
||||
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init");
|
||||
await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run");
|
||||
sub.Subscribed.ShouldBeEmpty();
|
||||
|
||||
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
|
||||
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed)
|
||||
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy
|
||||
|
||||
await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery");
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
|
||||
"the rediscovery must re-seed the probe set");
|
||||
platforms.Calls.ShouldBe(2);
|
||||
sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true);
|
||||
}
|
||||
|
||||
private sealed class ParkingPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public int ParkCall { get; init; }
|
||||
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Release() => _release.TrySetResult();
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var call = Interlocked.Increment(ref _calls);
|
||||
if (call == ParkCall)
|
||||
{
|
||||
Entered.TrySetResult();
|
||||
await _release.Task.WaitAsync(cancellationToken);
|
||||
return ["Plat01"]; // the pre-deploy view this pass fetched
|
||||
}
|
||||
return call == 1 ? [] : ["Plat01", "Eng02"];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts
|
||||
/// after it — otherwise the probe set would stay on the stale view until the next deploy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var deploys = new ChannelDeployWatchSource();
|
||||
var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
|
||||
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
|
||||
var raised = 0;
|
||||
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
|
||||
try
|
||||
{
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
|
||||
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap
|
||||
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
|
||||
(await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task,
|
||||
"the first rediscovery pass must be parked mid-fetch");
|
||||
|
||||
await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero));
|
||||
await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery");
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed");
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
|
||||
"the final probe set must reflect the latest deploy, not the parked pass's stale view");
|
||||
}
|
||||
|
||||
/// <summary>The production path (owned runtime) queries the platform source at init, with no DiscoverAsync.</summary>
|
||||
[Fact]
|
||||
public async Task Production_path_queries_the_platform_source_at_init()
|
||||
{
|
||||
var platforms = new ScriptedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms)
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => Task.CompletedTask,
|
||||
};
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init");
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
// ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before
|
||||
/// the subscriber is released) is refused and leaves no pump; before the shutdown the same
|
||||
/// call works (positive control).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the (background) seed must give the probe watcher its handle first");
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
|
||||
|
||||
sub.ParkUnsubscribe = true;
|
||||
var shutdown = driver.ShutdownAsync(CancellationToken.None);
|
||||
var subscribeCallsBefore = sub.SubscribeCalls;
|
||||
try
|
||||
{
|
||||
(await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task,
|
||||
"the teardown must be parked inside the probe watcher's unsubscribe");
|
||||
shutdown.IsCompleted.ShouldBeFalse();
|
||||
|
||||
// The pump is still live here (the teardown takes it after the probe watcher) — reusing it
|
||||
// is exactly the race: the call must be refused, not served on the dying subscriber.
|
||||
driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken");
|
||||
await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None));
|
||||
sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway");
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs.
|
||||
sub.ReleaseUnsubscribe();
|
||||
}
|
||||
await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown");
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
|
||||
}
|
||||
|
||||
// ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
|
||||
|
||||
/// <summary>A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline.</summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers()
|
||||
{
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource())
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => Task.CompletedTask,
|
||||
SessionCloseDeadline = TimeSpan.FromMilliseconds(300),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var closeToken = CancellationToken.None;
|
||||
driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct =>
|
||||
{
|
||||
closeToken = ct;
|
||||
closeEntered.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway
|
||||
};
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
sw.Stop();
|
||||
|
||||
closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted");
|
||||
closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s");
|
||||
driver.OwnedMxSessionForTests.ShouldBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
|
||||
}
|
||||
|
||||
// ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared
|
||||
/// registry; the same replay without a teardown rebinds (positive control).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind()
|
||||
{
|
||||
// Positive control first.
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription");
|
||||
sub.SubscribeCalls.ShouldBe(2);
|
||||
}
|
||||
|
||||
{
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
|
||||
var replay = driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
(await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task,
|
||||
"the replay must be parked inside its SubscribeBulk");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
|
||||
sub.ReleaseSubscribe();
|
||||
await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles");
|
||||
}
|
||||
}
|
||||
}
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Lane F fix round 2 (re-review of #735):
|
||||
/// <list type="bullet">
|
||||
/// <item>N1 — the init platform seed runs in the background: a slow or cancelled repository
|
||||
/// fetch can neither delay nor fail <c>InitializeAsync</c> (Galaxy's Tier-A Initialize budget is
|
||||
/// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown
|
||||
/// is discarded.</item>
|
||||
/// <item>N2 — a seed requested under a new runtime always gets a pass of its own, even when a
|
||||
/// worker from the previous runtime is still running.</item>
|
||||
/// <item>N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails
|
||||
/// and unsubscribes instead of registering into the next runtime's registry.</item>
|
||||
/// <item>The probe watcher's own subscribe re-checks its stop after the await.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverFixRound2Tests
|
||||
{
|
||||
private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>A platform source whose fetches park until released (optionally ignoring cancellation).</summary>
|
||||
private sealed class GatedPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public int ParkCall { get; init; } = 1;
|
||||
public bool IgnoreCancellation { get; init; }
|
||||
public Func<int, IReadOnlyList<string>> Result { get; init; } = _ => ["Plat01"];
|
||||
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource<CancellationToken> ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Release() => _release.TrySetResult();
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var call = Interlocked.Increment(ref _calls);
|
||||
if (call == ParkCall)
|
||||
{
|
||||
ParkedToken.TrySetResult(cancellationToken);
|
||||
Entered.TrySetResult();
|
||||
if (IgnoreCancellation) await _release.Task;
|
||||
else await _release.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
return Result(call);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
return Task.FromException<IReadOnlyList<string>>(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Entered(TaskCompletionSource tcs, string because) =>
|
||||
(await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because);
|
||||
|
||||
// ───────────────────────────── N1: background init seed ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy
|
||||
/// inside its budget, and the teardown cancels the parked fetch.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
try
|
||||
{
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
sw.Stop();
|
||||
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
await Entered(platforms.Entered, "positive control: the seed fetch was started");
|
||||
sw.Elapsed.ShouldBeLessThan(InitBudget);
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
var token = await platforms.ParkedToken.Task;
|
||||
token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight");
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A fetch that is cancelled (the gateway's own deadline) does not fail init.</summary>
|
||||
[Fact]
|
||||
public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline"));
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
|
||||
|
||||
// The driver keeps serving after the failed seed.
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The production failure mode: the host's Tier-A Initialize timeout cancels the init token
|
||||
/// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and
|
||||
/// the seed — tied to the runtime, not to the init call — must survive that cancellation and
|
||||
/// still land.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
using var initCts = new CancellationTokenSource();
|
||||
try
|
||||
{
|
||||
initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk
|
||||
await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
|
||||
await Entered(platforms.Entered, "the seed fetch must be in flight");
|
||||
await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked");
|
||||
(await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse(
|
||||
"the seed belongs to the runtime; the init call's timeout must not cancel it");
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the seed must still land after the init token was cancelled");
|
||||
}
|
||||
|
||||
/// <summary>Positive control: a fetch that completes after init still populates host statuses.</summary>
|
||||
[Fact]
|
||||
public async Task A_fetch_completing_after_init_populates_host_statuses()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
try
|
||||
{
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
await Entered(platforms.Entered, "the seed must be parked");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the late seed must reach the probe watcher");
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
|
||||
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
|
||||
"a ScanState event must surface the late-seeded platform");
|
||||
}
|
||||
|
||||
// ───────────────────────────── N1 generation + N2: straddling worker ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A seed from the previous runtime that completes after a teardown + re-init is discarded,
|
||||
/// and the new runtime's own seed request still gets its own pass.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource
|
||||
{
|
||||
IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it
|
||||
Result = call => call == 1 ? ["Stale01"] : ["Plat01"],
|
||||
};
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
try
|
||||
{
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
await Entered(platforms.Entered, "runtime 1's seed must be parked");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass");
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the new runtime must be seeded");
|
||||
driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]);
|
||||
lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded");
|
||||
}
|
||||
|
||||
// ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry
|
||||
/// back into the registry, and must release the handles it just acquired. Positive control:
|
||||
/// the same sync without a stop registers.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry()
|
||||
{
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control");
|
||||
}
|
||||
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
|
||||
var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry);
|
||||
|
||||
var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk");
|
||||
await watcher.StopAsync(CancellationToken.None);
|
||||
|
||||
sub.ReleaseSubscribe();
|
||||
await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register");
|
||||
watcher.WatchedPlatforms.ShouldBeEmpty();
|
||||
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"],
|
||||
"the handle acquired after the stop must be released");
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────────── N3: subscribe / read straddling a teardown ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing
|
||||
/// and unsubscribes its handles; a subscribe on the live runtime registers (positive control).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing()
|
||||
{
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers");
|
||||
|
||||
var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
try
|
||||
{
|
||||
await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk");
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sub.ReleaseSubscribe();
|
||||
}
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry");
|
||||
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry");
|
||||
}
|
||||
|
||||
/// <summary>The subscribe-once read path: same straddle, same outcome.</summary>
|
||||
[Fact]
|
||||
public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing()
|
||||
{
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None);
|
||||
try
|
||||
{
|
||||
await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk");
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
}
|
||||
finally
|
||||
{
|
||||
sub.ReleaseSubscribe();
|
||||
}
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]);
|
||||
}
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-4 / G-7 / G-11: the per-platform <c>ScanState</c> probing was structurally
|
||||
/// dead (its handles were never in the registry the pump resolves through, and a reconnect
|
||||
/// never re-subscribed them), discovery fetched the full galaxy twice, and
|
||||
/// <c>LastSuccessfulRead</c> never moved on data. These drive the real driver → pump → watcher
|
||||
/// → aggregator path with a fake subscriber and a fake hierarchy.
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverProbeWiringTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static GalaxyDriverOptions Opts() => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("ProbeWiring"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: false),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
private sealed class CountingHierarchySource : IGalaxyHierarchySource
|
||||
{
|
||||
public int Calls;
|
||||
|
||||
public Task<IReadOnlyList<GalaxyObject>> GetHierarchyAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref Calls);
|
||||
var platform = new GalaxyObject { GobjectId = 1, TagName = "Plat01", ContainedName = "Plat01" };
|
||||
platform.TemplateChain.Add("$WinPlatform");
|
||||
var tank = new GalaxyObject { GobjectId = 2, TagName = "Tank01", ContainedName = "Tank01", ParentGobjectId = 1 };
|
||||
tank.TemplateChain.Add("$UserDefined");
|
||||
tank.Attributes.Add(new GalaxyAttribute { AttributeName = "Level", FullTagReference = "Tank01.Level", MxDataType = 5 });
|
||||
return Task.FromResult<IReadOnlyList<GalaxyObject>>([platform, tank]);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StreamingSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle;
|
||||
|
||||
public List<string> Subscribed { get; } = [];
|
||||
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<SubscribeResult>();
|
||||
lock (Subscribed)
|
||||
{
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = Interlocked.Increment(ref _nextHandle);
|
||||
Subscribed.Add(r);
|
||||
HandleByRef[r] = h;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
}
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
||||
}
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { BoolValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
|
||||
public ValueTask EmitDoubleAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class NullBuilder : IAddressSpaceBuilder
|
||||
{
|
||||
public IAddressSpaceBuilder Folder(string browseName, string displayName) => this;
|
||||
public IVariableHandle Variable(string browseName, string displayName, DriverAttributeInfo attributeInfo) => new Handle(attributeInfo.FullName);
|
||||
public void AddProperty(string browseName, DriverDataType dataType, object? value) { }
|
||||
private sealed class Handle(string fullReference) : IVariableHandle
|
||||
{
|
||||
public string FullReference { get; } = fullReference;
|
||||
public IAlarmConditionSink MarkAsAlarmCondition(AlarmConditionInfo alarmInfo) => new NoopSink();
|
||||
}
|
||||
private sealed class NoopSink : IAlarmConditionSink
|
||||
{
|
||||
public void OnTransition(AlarmEventArgs args) { }
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
private static HostState? StateOf(GalaxyDriver driver, string host) =>
|
||||
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
|
||||
|
||||
/// <summary>
|
||||
/// A ScanState event on the probe handle updates GetHostStatuses; after a reconnect replay
|
||||
/// the probe is re-subscribed and events on the NEW handle keep updating it, while the dead
|
||||
/// pre-reconnect handle no longer does. Probe traffic never reaches public OnDataChange.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ScanState_event_updates_GetHostStatuses_and_survives_a_reconnect()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
var source = new CountingHierarchySource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
|
||||
var publicRefs = new List<string>();
|
||||
driver.OnDataChange += (_, e) => { lock (publicRefs) publicRefs.Add(e.FullReference); };
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
|
||||
// The pump starts on the first value subscription (the probe watcher does not start it).
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
|
||||
var probeHandle = sub.HandleByRef["Plat01.ScanState"];
|
||||
await sub.EmitBoolAsync(probeHandle, true);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
|
||||
"a ScanState=true event must surface Plat01 as Running");
|
||||
|
||||
// Reconnect replay: every pre-reconnect handle is dead; the probe must rebind.
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
var rebound = sub.HandleByRef["Plat01.ScanState"];
|
||||
rebound.ShouldNotBe(probeHandle, "replay must re-subscribe the probe and record the fresh handle");
|
||||
|
||||
await sub.EmitBoolAsync(rebound, false);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Stopped,
|
||||
"a ScanState=false event on the post-reconnect handle must surface Plat01 as Stopped");
|
||||
|
||||
// The dead handle is no longer routed (positive control: the live one still is).
|
||||
await sub.EmitBoolAsync(probeHandle, true);
|
||||
await sub.EmitDoubleAsync(sub.HandleByRef["Tank01.Level"], 1.0);
|
||||
await WaitUntilAsync(() => { lock (publicRefs) return publicRefs.Contains("Tank01.Level"); }, "value traffic must still flow");
|
||||
StateOf(driver, "Plat01").ShouldBe(HostState.Stopped, "an event on the pre-reconnect handle must be dropped");
|
||||
|
||||
lock (publicRefs) publicRefs.ShouldNotContain("Plat01.ScanState", "probe traffic is the watcher's, never the host's");
|
||||
}
|
||||
|
||||
/// <summary>Discovery asks the gateway for the hierarchy once; the probe set is selected from that same sweep.</summary>
|
||||
[Fact]
|
||||
public async Task DiscoverAsync_fetches_the_hierarchy_once_and_still_syncs_the_probe_set()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
var source = new CountingHierarchySource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
|
||||
|
||||
source.Calls.ShouldBe(1, "the probe selection must reuse the discoverer's sweep");
|
||||
sub.Subscribed.ShouldContain("Plat01.ScanState", "positive control: the platform probe was still synced");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
|
||||
}
|
||||
|
||||
/// <summary>Inbound stream data advances LastSuccessfulRead (throttled); the throttle holds the stamp still.</summary>
|
||||
[Fact]
|
||||
public async Task Stream_data_advances_LastSuccessfulRead_under_a_throttle()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
||||
{
|
||||
LivenessStampThrottle = TimeSpan.FromHours(1),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var initStamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
await Task.Delay(20);
|
||||
|
||||
var delivered = 0;
|
||||
driver.OnDataChange += (_, _) => Interlocked.Increment(ref delivered);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
var handle = sub.HandleByRef["Tank01.Level"];
|
||||
|
||||
await sub.EmitDoubleAsync(handle, 1.0);
|
||||
await WaitUntilAsync(() => Volatile.Read(ref delivered) == 1, "first event must be delivered");
|
||||
var first = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
first.ShouldBeGreaterThan(initStamp, "data on the stream is liveness evidence");
|
||||
|
||||
await Task.Delay(20);
|
||||
await sub.EmitDoubleAsync(handle, 2.0);
|
||||
await WaitUntilAsync(() => Volatile.Read(ref delivered) == 2, "second event must be delivered");
|
||||
driver.GetHealth().LastSuccessfulRead.ShouldBe(first, "inside the throttle window the stamp holds still");
|
||||
}
|
||||
}
|
||||
+386
@@ -0,0 +1,386 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="GalaxyDriver.ShutdownAsync"/> used to be a no-op (review 2026-10-21, G-1/G-2/G-9):
|
||||
/// the gRPC runtime outlived the driver on every respawn, removal and <c>/raw</c> browse, and
|
||||
/// the host's comms-loss recovery (<c>ShutdownAsync</c> → <c>InitializeAsync</c> on the SAME
|
||||
/// object) neither tore down nor rebuilt anything. These tests pin the teardown at every
|
||||
/// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed,
|
||||
/// the owned channels + session — and the rebuild on the next <c>InitializeAsync</c>.
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverShutdownTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("ShutdownTest"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
// ───────────────────────────── fakes ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Subscriber whose event stream records the token it was opened with and completes a
|
||||
/// signal when that token fires — the observable for "the StreamEvents stream was cancelled".
|
||||
/// </summary>
|
||||
private sealed class StreamingSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle;
|
||||
|
||||
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public List<int> Unsubscribed { get; } = [];
|
||||
public int StreamsOpened;
|
||||
public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<SubscribeResult>();
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = Interlocked.Increment(ref _nextHandle);
|
||||
HandleByRef[r] = h;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
||||
}
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref StreamsOpened);
|
||||
using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult());
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when
|
||||
/// the enumerator is cancelled or completes) — the observable for "exactly one watcher".
|
||||
/// </summary>
|
||||
private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource
|
||||
{
|
||||
private int _open;
|
||||
private int _opened;
|
||||
public Channel<DeployEvent> Current { get; private set; } = Channel.CreateUnbounded<DeployEvent>();
|
||||
|
||||
public int Open => Volatile.Read(ref _open);
|
||||
public int Opened => Volatile.Read(ref _opened);
|
||||
|
||||
public async IAsyncEnumerable<DeployEvent> WatchAsync(
|
||||
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _open);
|
||||
Interlocked.Increment(ref _opened);
|
||||
var channel = Current = Channel.CreateUnbounded<DeployEvent>();
|
||||
try
|
||||
{
|
||||
await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref _open);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent
|
||||
{
|
||||
Sequence = sequence,
|
||||
TimeOfLastDeployPresent = true,
|
||||
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Canned platform set so a production-path init never dials the (fake) gateway for one.</summary>
|
||||
private sealed class NoPlatforms : IGalaxyPlatformSource
|
||||
{
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<string>>([]);
|
||||
}
|
||||
|
||||
private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed
|
||||
{
|
||||
public int Starts;
|
||||
public int Disposes;
|
||||
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
|
||||
public void Start() => Starts++;
|
||||
public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition(
|
||||
"Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise,
|
||||
AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi"));
|
||||
public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; }
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────────── G-1: streams + pump ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the
|
||||
/// registry is empty. Positive control: an event delivered through that same stream
|
||||
/// reached OnDataChange before the shutdown.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
driver.OnDataChange += (_, e) => delivered.TrySetResult(e);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeTrue();
|
||||
|
||||
await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0);
|
||||
(await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task,
|
||||
"positive control: the stream must deliver before the shutdown");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
// Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the
|
||||
// enumerator's cancellation registration has run by the time it returns (observed flaky).
|
||||
(await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task,
|
||||
"ShutdownAsync must cancel the StreamEvents stream");
|
||||
driver.HasEventPumpForTests.ShouldBeFalse();
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init");
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
|
||||
driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT
|
||||
/// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive
|
||||
/// control on both sides of the shutdown.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it()
|
||||
{
|
||||
var feed = new RecordingAlarmFeed();
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
|
||||
subscriber: sub, alarmFeed: feed);
|
||||
var received = 0;
|
||||
driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
|
||||
feed.Emit();
|
||||
received.ShouldBe(1, "positive control: the feed is wired before the shutdown");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
feed.Emit();
|
||||
received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent");
|
||||
feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector");
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
|
||||
feed.Emit();
|
||||
received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed");
|
||||
feed.Starts.ShouldBe(2);
|
||||
}
|
||||
|
||||
// ───────────────────────────── G-2: one DeployWatcher ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch
|
||||
/// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly
|
||||
/// once — five stacked watchers would raise five times.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher()
|
||||
{
|
||||
var source = new CountingDeployWatchSource();
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
deployWatchSource: source);
|
||||
var raised = 0;
|
||||
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
for (var cycle = 0; cycle < 5; cycle++)
|
||||
{
|
||||
// Let this cycle's watch stream actually open before the next shutdown, so every cycle
|
||||
// holds a live stream the teardown has to cancel (a loop cancelled before its first
|
||||
// open would make the count vacuous).
|
||||
await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open");
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1,
|
||||
$"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}");
|
||||
driver.DeployWatcherForTests.ShouldNotBeNull();
|
||||
|
||||
// Bootstrap (suppressed) then a genuine change.
|
||||
await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
|
||||
await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
|
||||
await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery");
|
||||
await Task.Delay(200);
|
||||
Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream");
|
||||
driver.DeployWatcherForTests.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one.</summary>
|
||||
[Fact]
|
||||
public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher()
|
||||
{
|
||||
var source = new CountingDeployWatchSource();
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
deployWatchSource: source);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var first = driver.DeployWatcherForTests;
|
||||
await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open");
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1,
|
||||
$"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}");
|
||||
driver.DeployWatcherForTests.ShouldNotBeSameAs(first);
|
||||
}
|
||||
|
||||
// ───────────────────────────── G-1/G-9: owned runtime, channels + session ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// The production path (no injected seams): InitializeAsync builds the owned client, session
|
||||
/// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the
|
||||
/// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's
|
||||
/// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC
|
||||
/// channels are real but lazy (nothing dials until a call is made).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds()
|
||||
{
|
||||
var opens = 0;
|
||||
var source = new CountingDeployWatchSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source,
|
||||
platformSource: new NoPlatforms())
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; },
|
||||
};
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
opens.ShouldBe(1);
|
||||
var client = driver.OwnedMxClientForTests.ShouldNotBeNull();
|
||||
var session = driver.OwnedMxSessionForTests.ShouldNotBeNull();
|
||||
session.IsConnected.ShouldBeTrue();
|
||||
driver.SupervisorForTests.ShouldNotBeNull();
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull();
|
||||
driver.DeployWatcherForTests.ShouldNotBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session");
|
||||
await Should.ThrowAsync<ObjectDisposedException>(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None),
|
||||
"the owned gateway channel must be disposed");
|
||||
driver.OwnedMxClientForTests.ShouldBeNull();
|
||||
driver.OwnedMxSessionForTests.ShouldBeNull();
|
||||
driver.SupervisorForTests.ShouldBeNull();
|
||||
driver.ProbeWatcherForTests.ShouldBeNull();
|
||||
driver.DeployWatcherForTests.ShouldBeNull();
|
||||
await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled");
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
|
||||
driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp);
|
||||
|
||||
// The watchdog's second half: a fresh runtime on the same object.
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one");
|
||||
driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client);
|
||||
driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session);
|
||||
driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue();
|
||||
driver.SupervisorForTests.ShouldNotBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A connect that fails mid-build releases the partial runtime (no orphaned client per
|
||||
/// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds()
|
||||
{
|
||||
var attempts = 0;
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
|
||||
platformSource: new NoPlatforms())
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => ++attempts == 1
|
||||
? throw new InvalidOperationException("gateway refused")
|
||||
: Task.CompletedTask,
|
||||
};
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(() => driver.InitializeAsync("{}", CancellationToken.None));
|
||||
ex.Message.ShouldBe("gateway refused");
|
||||
driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client");
|
||||
driver.OwnedMxSessionForTests.ShouldBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Faulted);
|
||||
driver.GetHealth().LastError.ShouldBe("gateway refused");
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
attempts.ShouldBe(2);
|
||||
driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
/// <summary>DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op.</summary>
|
||||
[Fact]
|
||||
public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
await Should.NotThrowAsync(async () => await driver.DisposeAsync());
|
||||
await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None));
|
||||
await Should.ThrowAsync<ObjectDisposedException>(() => driver.InitializeAsync("{}", CancellationToken.None));
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="GalaxyMxSession.CloseAsync"/> bounds the gateway <c>CloseSession</c> round-trip by
|
||||
/// the caller's token (the SDK dispose closes under its 30 s call timeout) and leaves the session
|
||||
/// re-openable.
|
||||
/// </summary>
|
||||
public sealed class GalaxyMxSessionCloseTests
|
||||
{
|
||||
/// <summary>A close against a never-answering gateway returns when the token fires, without throwing.</summary>
|
||||
[Fact]
|
||||
public async Task CloseAsync_returns_at_the_token_deadline_against_a_never_answering_gateway()
|
||||
{
|
||||
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = ct => Task.Delay(Timeout.Infinite, ct),
|
||||
};
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
session.IsConnected.ShouldBeTrue();
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
await session.CloseAsync(cts.Token).WaitAsync(TimeSpan.FromSeconds(10));
|
||||
sw.Stop();
|
||||
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5));
|
||||
session.IsConnected.ShouldBeFalse();
|
||||
|
||||
// Still usable: a close is not a dispose.
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
session.IsConnected.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>Positive control: a close that answers completes and reports the token it was handed.</summary>
|
||||
[Fact]
|
||||
public async Task CloseAsync_completes_when_the_gateway_answers()
|
||||
{
|
||||
CancellationToken seen = default;
|
||||
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = ct => { seen = ct; return Task.CompletedTask; },
|
||||
};
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
await session.CloseAsync(cts.Token);
|
||||
|
||||
seen.ShouldBe(cts.Token, "the caller's token must reach the gateway round-trip");
|
||||
session.IsConnected.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The driver's teardown is CloseAsync (bounded) then DisposeAsync (final). The dispose must
|
||||
/// not issue a second gateway close; a fresh connect after a close must close again (positive
|
||||
/// control: the guard keys on "was connected", not on "ever closed").
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CloseAsync_then_DisposeAsync_closes_the_gateway_session_once()
|
||||
{
|
||||
var closes = 0;
|
||||
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
|
||||
};
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
|
||||
await session.CloseAsync(CancellationToken.None);
|
||||
await session.DisposeAsync();
|
||||
closes.ShouldBe(1, "dispose after a close must not close the gateway session again");
|
||||
|
||||
var reopened = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
|
||||
};
|
||||
await reopened.ConnectAsync(null!, CancellationToken.None);
|
||||
await reopened.CloseAsync(CancellationToken.None);
|
||||
await reopened.ConnectAsync(null!, CancellationToken.None);
|
||||
await reopened.DisposeAsync();
|
||||
closes.ShouldBe(3, "each live session closes exactly once");
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
/// <see cref="ActivityListener"/> rather than asserting on internal state, so the
|
||||
/// tests double as documentation of the listener-side contract.
|
||||
/// </summary>
|
||||
[Collection(ProcessWideTelemetryCollection.Name)]
|
||||
public sealed class GalaxyTelemetryTests
|
||||
{
|
||||
/// <summary>Subscribes an ActivityListener for the test, captures each spawned activity.</summary>
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-8: a batch of writes ran strictly serially, three round-trips per
|
||||
/// first-touch tag. Distinct tags now run as concurrent lanes bounded at
|
||||
/// <see cref="GatewayGalaxyDataWriter.MaxConcurrentWriteLanes"/>; same-tag writes keep their
|
||||
/// order inside one lane; results stay at their request's index.
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyDataWriterConcurrencyTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static GatewayGalaxyDataWriter NewWriter()
|
||||
=> new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0);
|
||||
|
||||
/// <summary>
|
||||
/// Fake ops whose AddItem parks every caller until <see cref="ReleaseAt"/> callers are in
|
||||
/// flight at once (or a safety delay elapses), recording the peak concurrency.
|
||||
/// </summary>
|
||||
private sealed class GatingOps : IMxWriteOps
|
||||
{
|
||||
private int _inFlight;
|
||||
private int _peak;
|
||||
private int _nextHandle;
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly ConcurrentDictionary<string, int> _handles = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public int ReleaseAt { get; init; } = int.MaxValue;
|
||||
public int Peak => Volatile.Read(ref _peak);
|
||||
public ConcurrentQueue<(int Handle, MxValue Value)> RawWrites { get; } = new();
|
||||
public ConcurrentQueue<string> AddItems { get; } = new();
|
||||
public Func<int, MxCommandReply>? WriteResponder { get; init; }
|
||||
|
||||
/// <summary>Deterministic handle per tag when set; otherwise handles are assigned in AddItem completion order.</summary>
|
||||
public Func<string, int>? HandleFor { get; init; }
|
||||
|
||||
public string SessionId => "sess";
|
||||
public int ServerHandle => 1;
|
||||
|
||||
public async Task<int> AddItemAsync(string fullRef, CancellationToken ct)
|
||||
{
|
||||
AddItems.Enqueue(fullRef);
|
||||
var now = Interlocked.Increment(ref _inFlight);
|
||||
int seen;
|
||||
do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen);
|
||||
if (now >= ReleaseAt) _release.TrySetResult();
|
||||
// Park on the signal; the delay is only a safety net against a wedged test (a starved
|
||||
// pool must not release the gate early and turn the peak assertion flaky).
|
||||
await Task.WhenAny(_release.Task, Task.Delay(WaitMs, ct));
|
||||
Interlocked.Decrement(ref _inFlight);
|
||||
return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle));
|
||||
}
|
||||
|
||||
public Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken ct)
|
||||
=> Task.FromResult(new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } });
|
||||
|
||||
public Task<MxCommandReply> WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct)
|
||||
{
|
||||
RawWrites.Enqueue((itemHandle, value));
|
||||
return Task.FromResult(WriteResponder?.Invoke(itemHandle) ?? GoodStatusRow());
|
||||
}
|
||||
|
||||
// A reply carrying a Good MX status ROW (not an empty statuses array): the writer's static
|
||||
// meters are process-wide, and an empty-statuses Good would increment galaxy.writes.unconfirmed
|
||||
// under the meter tests running in parallel in another class.
|
||||
private static MxCommandReply GoodStatusRow()
|
||||
{
|
||||
var reply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } };
|
||||
reply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok });
|
||||
return reply;
|
||||
}
|
||||
}
|
||||
|
||||
private static SecurityClassification Free(string _) => SecurityClassification.FreeAccess;
|
||||
|
||||
/// <summary>Sixteen distinct tags: lanes overlap (not serial) and never exceed the cap of eight.</summary>
|
||||
[Fact]
|
||||
public async Task Distinct_tags_write_concurrently_bounded_at_the_lane_cap()
|
||||
{
|
||||
var ops = new GatingOps { ReleaseAt = GatewayGalaxyDataWriter.MaxConcurrentWriteLanes };
|
||||
var writes = Enumerable.Range(0, 16).Select(i => new WriteRequest($"Tag{i}.PV", (double)i)).ToArray();
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
ops.Peak.ShouldBe(GatewayGalaxyDataWriter.MaxConcurrentWriteLanes,
|
||||
"first-touch lanes must overlap (serial = 1) and stay bounded (unbounded > 8)");
|
||||
results.Count.ShouldBe(16);
|
||||
results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
|
||||
ops.AddItems.Count.ShouldBe(16, "one AddItem per distinct tag");
|
||||
ops.RawWrites.Count.ShouldBe(16);
|
||||
}
|
||||
|
||||
/// <summary>Writes to the same tag stay in input order inside their lane and share one AddItem.</summary>
|
||||
[Fact]
|
||||
public async Task Same_tag_writes_stay_ordered_and_share_one_handle()
|
||||
{
|
||||
var ops = new GatingOps { ReleaseAt = 2 };
|
||||
var writes = new[]
|
||||
{
|
||||
new WriteRequest("Tank01.SP", 1.0),
|
||||
new WriteRequest("Tank02.SP", 10.0),
|
||||
new WriteRequest("Tank01.SP", 2.0),
|
||||
new WriteRequest("Tank01.SP", 3.0),
|
||||
};
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
results.Count.ShouldBe(4);
|
||||
results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
|
||||
ops.AddItems.Count(a => a == "Tank01.SP").ShouldBe(1, "a lane AddItems its tag once");
|
||||
var tank01 = ops.RawWrites.Where(w => w.Handle == ops.RawWrites.First(x => x.Value.DoubleValue == 1.0).Handle)
|
||||
.Select(w => w.Value.DoubleValue).ToArray();
|
||||
tank01.ShouldBe([1.0, 2.0, 3.0], "same-tag writes must keep their authored order");
|
||||
}
|
||||
|
||||
/// <summary>A per-tag failure lands at that request's index while the other lanes stay Good.</summary>
|
||||
[Fact]
|
||||
public async Task Results_land_at_their_request_index()
|
||||
{
|
||||
var ops = new GatingOps
|
||||
{
|
||||
ReleaseAt = 3,
|
||||
HandleFor = tag => tag[0] - 'A' + 1, // A.PV → 1, B.PV → 2, C.PV → 3
|
||||
WriteResponder = handle => new MxCommandReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus
|
||||
{
|
||||
Code = handle == 2 ? ProtocolStatusCode.MxaccessFailure : ProtocolStatusCode.Ok,
|
||||
},
|
||||
},
|
||||
};
|
||||
var writes = new[]
|
||||
{
|
||||
new WriteRequest("A.PV", 1.0),
|
||||
new WriteRequest("B.PV", 2.0), // refused by the gateway
|
||||
new WriteRequest("C.PV", 3.0),
|
||||
};
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
results.Select(r => r.StatusCode).ShouldBe(
|
||||
[StatusCodeMap.Good, StatusCodeMap.BadCommunicationError, StatusCodeMap.Good],
|
||||
"each result must sit at its request's index regardless of lane completion order");
|
||||
}
|
||||
|
||||
/// <summary>A single-tag batch takes the direct path and still produces one result per request.</summary>
|
||||
[Fact]
|
||||
public async Task Single_tag_batch_runs_serially_in_order()
|
||||
{
|
||||
var ops = new GatingOps { ReleaseAt = 1 };
|
||||
var writes = new[] { new WriteRequest("X.PV", 1.0), new WriteRequest("X.PV", 2.0) };
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
results.Count.ShouldBe(2);
|
||||
ops.Peak.ShouldBe(1);
|
||||
ops.RawWrites.Select(w => w.Value.DoubleValue).ShouldBe([1.0, 2.0]);
|
||||
}
|
||||
}
|
||||
+1
@@ -13,6 +13,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
/// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal
|
||||
/// <see cref="IMxWriteOps"/> seam (the SDK session types are sealed + internal-ctor and cannot be faked).
|
||||
/// </summary>
|
||||
[Collection(ProcessWideTelemetryCollection.Name)]
|
||||
public sealed class GatewayGalaxyDataWriterFailClosedTests
|
||||
{
|
||||
private static GatewayGalaxyDataWriter NewWriter()
|
||||
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-3: <see cref="ReconnectSupervisor.ReportTransportFailure"/> allocated a
|
||||
/// new <see cref="CancellationTokenSource"/> per recovery cycle and only the last one was ever
|
||||
/// disposed. Each cycle now releases its own CTS when its loop exits, and
|
||||
/// <see cref="ReconnectSupervisor.StopAsync"/> awaits the loop instead of blocking on it.
|
||||
/// </summary>
|
||||
public sealed class ReconnectSupervisorCycleCtsTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static ReconnectOptions FastBackoff() =>
|
||||
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
|
||||
|
||||
/// <summary>Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control).</summary>
|
||||
[Fact]
|
||||
public async Task Every_completed_recovery_cycle_releases_its_own_cts()
|
||||
{
|
||||
var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var hold = new[] { false }; // read on the recovery loop's thread, written here — Volatile both ways
|
||||
using var sup = new ReconnectSupervisor(
|
||||
reopen: async _ =>
|
||||
{
|
||||
if (Volatile.Read(ref hold[0]))
|
||||
{
|
||||
reopenEntered.TrySetResult();
|
||||
await holdReopen.Task;
|
||||
}
|
||||
},
|
||||
replay: _ => Task.CompletedTask,
|
||||
options: FastBackoff());
|
||||
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
sup.ReportTransportFailure(new IOException($"drop {i}"));
|
||||
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
|
||||
}
|
||||
await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS");
|
||||
sup.CyclesStarted.ShouldBe(3);
|
||||
|
||||
// Positive control: a cycle parked inside reopen still holds its CTS.
|
||||
Volatile.Write(ref hold[0], true);
|
||||
sup.ReportTransportFailure(new IOException("drop 3"));
|
||||
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
|
||||
sup.CyclesStarted.ShouldBe(4);
|
||||
sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits");
|
||||
|
||||
holdReopen.TrySetResult();
|
||||
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
|
||||
await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit");
|
||||
}
|
||||
|
||||
/// <summary>StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking.</summary>
|
||||
[Fact]
|
||||
public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts()
|
||||
{
|
||||
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var sup = new ReconnectSupervisor(
|
||||
reopen: async ct =>
|
||||
{
|
||||
reopenEntered.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
},
|
||||
replay: _ => Task.CompletedTask,
|
||||
options: FastBackoff());
|
||||
|
||||
sup.ReportTransportFailure(new IOException("drop"));
|
||||
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
|
||||
|
||||
var stop = sup.StopAsync();
|
||||
(await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return");
|
||||
sup.CyclesReleased.ShouldBe(1);
|
||||
await Should.NotThrowAsync(sup.StopAsync); // idempotent
|
||||
Should.NotThrow(sup.Dispose);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user