Merge lane F: Galaxy driver teardown, probing, knobs (#735)
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
+92
@@ -0,0 +1,92 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-5: <c>Repository.DiscoverPageSize</c> was authored and parsed but reached
|
||||
/// nothing — the client's typed <c>DiscoverHierarchyAsync</c> hard-codes 5000. The source now pages
|
||||
/// the raw RPC itself at the configured size.
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyHierarchySourcePagingTests
|
||||
{
|
||||
private static GalaxyObject Obj(int id) => new() { GobjectId = id, TagName = $"Obj{id}" };
|
||||
|
||||
/// <summary>Every request carries the configured page size and the page tokens chain until the last page.</summary>
|
||||
[Fact]
|
||||
public async Task Pages_at_the_configured_size_and_chains_tokens()
|
||||
{
|
||||
var requests = new List<DiscoverHierarchyRequest>();
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
requests.Add(req);
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
switch (req.PageToken)
|
||||
{
|
||||
case "":
|
||||
reply.Objects.Add([Obj(1), Obj(2)]);
|
||||
reply.NextPageToken = "p2";
|
||||
break;
|
||||
case "p2":
|
||||
reply.Objects.Add([Obj(3), Obj(4)]);
|
||||
reply.NextPageToken = "p3";
|
||||
break;
|
||||
default:
|
||||
reply.Objects.Add(Obj(5));
|
||||
reply.NextPageToken = "";
|
||||
break;
|
||||
}
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var objects = await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 2, CancellationToken.None);
|
||||
|
||||
objects.Select(o => o.GobjectId).ShouldBe([1, 2, 3, 4, 5]);
|
||||
requests.Count.ShouldBe(3);
|
||||
requests.ShouldAllBe(r => r.PageSize == 2);
|
||||
requests.Select(r => r.PageToken).ShouldBe(["", "p2", "p3"]);
|
||||
}
|
||||
|
||||
/// <summary>A different configured size is what goes on the wire (positive control for the knob).</summary>
|
||||
[Fact]
|
||||
public async Task A_different_page_size_is_sent_on_the_wire()
|
||||
{
|
||||
int? sent = null;
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
sent = req.PageSize;
|
||||
return Task.FromResult(new DiscoverHierarchyReply());
|
||||
}
|
||||
|
||||
await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 250, CancellationToken.None);
|
||||
sent.ShouldBe(250);
|
||||
}
|
||||
|
||||
/// <summary>A non-positive configured size falls back to the client library's 5000 default.</summary>
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-7)]
|
||||
public void Non_positive_page_size_falls_back_to_the_default(int configured)
|
||||
=> GatewayGalaxyHierarchySource.EffectivePageSize(configured).ShouldBe(GatewayGalaxyHierarchySource.DefaultPageSize);
|
||||
|
||||
/// <summary>A repeated page token is refused rather than walked forever.</summary>
|
||||
[Fact]
|
||||
public async Task A_repeated_page_token_is_refused()
|
||||
{
|
||||
var calls = 0;
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
calls++;
|
||||
var reply = new DiscoverHierarchyReply { NextPageToken = "same" };
|
||||
reply.Objects.Add(Obj(calls));
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 10, CancellationToken.None));
|
||||
ex.Message.ShouldContain("repeated page token");
|
||||
calls.ShouldBe(2);
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
|
||||
|
||||
/// <summary>The platform-only query the driver seeds its ScanState probe watcher from.</summary>
|
||||
public sealed class GatewayGalaxyPlatformSourceTests
|
||||
{
|
||||
private static GalaxyObject Obj(string tag, params string[] chain)
|
||||
{
|
||||
var o = new GalaxyObject { TagName = tag };
|
||||
o.TemplateChain.Add(chain);
|
||||
return o;
|
||||
}
|
||||
|
||||
/// <summary>Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered.</summary>
|
||||
[Fact]
|
||||
public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered()
|
||||
{
|
||||
var requests = new List<DiscoverHierarchyRequest>();
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
requests.Add(req);
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
if (req.PageToken == "")
|
||||
{
|
||||
reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]);
|
||||
reply.NextPageToken = "p2";
|
||||
}
|
||||
else
|
||||
{
|
||||
reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]);
|
||||
}
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None);
|
||||
|
||||
names.ShouldBe(["Plat01", "Eng01"]);
|
||||
requests.Count.ShouldBe(2);
|
||||
requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false);
|
||||
requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine"));
|
||||
}
|
||||
|
||||
/// <summary>A gateway that strips the template chain is trusted to have filtered server-side.</summary>
|
||||
[Fact]
|
||||
public async Task Objects_without_a_template_chain_are_kept()
|
||||
{
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
reply.Objects.Add(new GalaxyObject { TagName = "Plat01" });
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
(await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 0, CancellationToken.None)).ShouldBe(["Plat01"]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-6: <c>ProbeTimeoutSeconds</c> was authored and compared for equivalence but
|
||||
/// never consumed — the probe ran on the caller's budget alone. It now caps the probe deadline
|
||||
/// (it can shorten the caller's budget, never extend it).
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class GalaxyDriverProbeTimeoutTests
|
||||
{
|
||||
/// <summary>The effective deadline is the shorter of the caller's budget and the configured cap.</summary>
|
||||
[Theory]
|
||||
[InlineData(10, 30, 10)] // caller shorter → caller wins
|
||||
[InlineData(10, 2, 2)] // config shorter → config caps
|
||||
[InlineData(10, 0, 10)] // unset config → caller
|
||||
[InlineData(10, -1, 10)] // nonsense config → caller
|
||||
[InlineData(0, 5, 1)] // no caller budget → 1 s floor beats a longer config
|
||||
[InlineData(0, 0, 1)] // nothing → 1 s floor
|
||||
public void EffectiveTimeout_is_the_shorter_of_caller_and_config(int callerSeconds, int configSeconds, int expectedSeconds)
|
||||
=> GalaxyDriverProbe.EffectiveTimeout(TimeSpan.FromSeconds(callerSeconds), configSeconds)
|
||||
.ShouldBe(TimeSpan.FromSeconds(expectedSeconds));
|
||||
|
||||
/// <summary>
|
||||
/// Against a port that accepts TCP but never speaks gRPC, a 1 s configured cap returns long
|
||||
/// before the caller's 20 s budget. The gRPC handshake itself does the waiting here, so this
|
||||
/// exercises the real deadline path, not the pure helper.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Configured_cap_shortens_the_probe_against_a_silent_listener()
|
||||
{
|
||||
var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
try
|
||||
{
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
var config = $$"""
|
||||
{
|
||||
"Gateway": { "Endpoint": "http://127.0.0.1:{{port}}", "ApiKeySecretRef": "dev:k", "UseTls": false },
|
||||
"MxAccess": { "ClientName": "probe-timeout" },
|
||||
"Repository": {},
|
||||
"Reconnect": {},
|
||||
"ProbeTimeoutSeconds": 1
|
||||
}
|
||||
""";
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = await new GalaxyDriverProbe().ProbeAsync(config, TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
|
||||
sw.Stop();
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), $"the 1 s configured cap must bound the probe (message: {result.Message})");
|
||||
result.Message.ShouldNotBeNull();
|
||||
(result.Message!.Contains("timed out", StringComparison.OrdinalIgnoreCase)
|
||||
|| result.Message.Contains("DeadlineExceeded", StringComparison.OrdinalIgnoreCase)
|
||||
|| result.Message.Contains("handshake failed", StringComparison.OrdinalIgnoreCase))
|
||||
.ShouldBeTrue($"unexpected message: '{result.Message}'");
|
||||
}
|
||||
finally
|
||||
{
|
||||
listener.Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-4: the watcher's probe handles must live in the driver's
|
||||
/// <see cref="SubscriptionRegistry"/> under the reserved id, follow the platform set, be
|
||||
/// rebound on <see cref="PerPlatformProbeWatcher.ResubscribeAsync"/>, and leave on stop.
|
||||
/// </summary>
|
||||
public sealed class PerPlatformProbeWatcherRegistryTests
|
||||
{
|
||||
private sealed class FakeSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private int _next;
|
||||
public List<int[]> Unsubscribes { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<SubscribeResult>>(
|
||||
[.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = ++_next, WasSuccessful = true })]);
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
Unsubscribes.Add([.. itemHandles]);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, cancellationToken);
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
|
||||
private static (long Id, string Ref)[] Resolve(SubscriptionRegistry registry, int handle) =>
|
||||
[.. registry.ResolveSubscribers(handle)];
|
||||
|
||||
/// <summary>Synced platforms resolve through the registry under the reserved id; removed ones stop resolving.</summary>
|
||||
[Fact]
|
||||
public async Task Sync_registers_probe_handles_under_the_reserved_id_and_tracks_removals()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
|
||||
|
||||
await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
|
||||
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue();
|
||||
var bindings = watcher.CurrentBindings;
|
||||
bindings.Count.ShouldBe(2);
|
||||
foreach (var b in bindings)
|
||||
{
|
||||
Resolve(registry, b.ItemHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, b.FullReference)]);
|
||||
}
|
||||
|
||||
var bHandle = bindings.Single(b => b.FullReference == "B.ScanState").ItemHandle;
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
Resolve(registry, bHandle).ShouldBeEmpty("a removed platform's handle must leave the fan-out map");
|
||||
var aHandle = bindings.Single(b => b.FullReference == "A.ScanState").ItemHandle;
|
||||
Resolve(registry, aHandle).ShouldHaveSingleItem("positive control: the kept platform still resolves");
|
||||
}
|
||||
|
||||
/// <summary>Resubscribe replaces the handles and rebinds: old handles stop resolving, new ones resolve.</summary>
|
||||
[Fact]
|
||||
public async Task Resubscribe_rebinds_the_registry_to_the_fresh_handles()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
var oldHandle = watcher.CurrentBindings.Single().ItemHandle;
|
||||
|
||||
(await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
|
||||
|
||||
var newHandle = watcher.CurrentBindings.Single().ItemHandle;
|
||||
newHandle.ShouldNotBe(oldHandle);
|
||||
Resolve(registry, oldHandle).ShouldBeEmpty();
|
||||
Resolve(registry, newHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, "A.ScanState")]);
|
||||
registry.TrackedSubscriptionCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>StopAsync unsubscribes the live handles and removes the reserved entry.</summary>
|
||||
[Fact]
|
||||
public async Task StopAsync_unsubscribes_and_removes_the_reserved_entry()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var subscriber = new FakeSubscriber();
|
||||
var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None);
|
||||
var handles = watcher.CurrentBindings.Select(b => b.ItemHandle).OrderBy(h => h).ToArray();
|
||||
|
||||
await watcher.StopAsync(CancellationToken.None);
|
||||
|
||||
subscriber.Unsubscribes.ShouldHaveSingleItem().OrderBy(h => h).ShouldBe(handles);
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse();
|
||||
registry.TrackedItemHandleCount.ShouldBe(0);
|
||||
await Should.NotThrowAsync(() => watcher.StopAsync(CancellationToken.None)); // idempotent
|
||||
}
|
||||
|
||||
/// <summary>A watcher without a registry (the decoding-table unit tests) still works and registers nothing.</summary>
|
||||
[Fact]
|
||||
public async Task Without_a_registry_the_watcher_keeps_its_handles_private()
|
||||
{
|
||||
using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator());
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
watcher.CurrentBindings.ShouldHaveSingleItem();
|
||||
(await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Classes that assert on process-wide telemetry (an <c>ActivityListener</c> on the Galaxy
|
||||
/// ActivitySource, a <c>MeterListener</c> on the Galaxy meter) run with parallelization disabled:
|
||||
/// those listeners see every Galaxy span / measurement in the process, so any parallel class that
|
||||
/// drives a traced subscriber or the data writer — including a driver's background probe seed that
|
||||
/// outlives its test — pollutes their counts (F-FU6, observed flaky in lane F fix round 2).
|
||||
/// </summary>
|
||||
[CollectionDefinition(Name, DisableParallelization = true)]
|
||||
public sealed class ProcessWideTelemetryCollection
|
||||
{
|
||||
/// <summary>The collection name.</summary>
|
||||
public const string Name = "Process-wide Galaxy telemetry";
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-10: a dispatch-loop exit left the read loop draining into a channel
|
||||
/// nobody read, with health Healthy and nothing telling the supervisor — a clean termination
|
||||
/// evading the watchdog. The exit now reports a stream fault. The fault is injected through
|
||||
/// the pump's handle factory, the one dispatch-side seam that runs outside the per-handler
|
||||
/// isolation.
|
||||
/// </summary>
|
||||
public sealed class EventPumpDispatchFaultTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private sealed class ChannelSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<SubscribeResult>>([]);
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitAsync(int handle) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = 1 },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>A dispatch-loop exception reports a stream fault carrying that exception.</summary>
|
||||
[Fact]
|
||||
public async Task Dispatch_loop_exit_reports_a_stream_fault()
|
||||
{
|
||||
var subscriber = new ChannelSubscriber();
|
||||
var registry = new SubscriptionRegistry();
|
||||
registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
|
||||
var fault = new TaskCompletionSource<Exception>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
await using var pump = new EventPump(
|
||||
subscriber, registry,
|
||||
handleFactory: _ => throw new InvalidDataException("synthetic dispatch failure"),
|
||||
channelCapacity: 4, clientName: "DispatchFault",
|
||||
onStreamFault: ex => fault.TrySetResult(ex));
|
||||
pump.Start();
|
||||
|
||||
await subscriber.EmitAsync(7);
|
||||
|
||||
(await Task.WhenAny(fault.Task, Task.Delay(WaitMs))).ShouldBe(fault.Task,
|
||||
"a dispatch-loop exit must be reported as a stream fault");
|
||||
(await fault.Task).ShouldBeOfType<InvalidDataException>();
|
||||
}
|
||||
|
||||
/// <summary>Positive control: the same event with a working handle factory dispatches and reports no fault.</summary>
|
||||
[Fact]
|
||||
public async Task Same_event_with_a_working_factory_dispatches_without_a_fault()
|
||||
{
|
||||
var subscriber = new ChannelSubscriber();
|
||||
var registry = new SubscriptionRegistry();
|
||||
registry.Register(1, [new TagBinding("Tank01.Level", 7)]);
|
||||
var faulted = false;
|
||||
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
await using var pump = new EventPump(
|
||||
subscriber, registry, channelCapacity: 4, clientName: "DispatchFault",
|
||||
onStreamFault: _ => faulted = true);
|
||||
pump.OnDataChange += (_, e) => delivered.TrySetResult(e);
|
||||
pump.Start();
|
||||
|
||||
await subscriber.EmitAsync(7);
|
||||
|
||||
(await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task);
|
||||
(await delivered.Task).FullReference.ShouldBe("Tank01.Level");
|
||||
faulted.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
+406
@@ -0,0 +1,406 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Lane F fix round 1 (review of #735): the probe watcher is seeded at <c>InitializeAsync</c>
|
||||
/// and re-seeded on rediscovery (the runtime never calls <c>DiscoverAsync</c> on a live driver);
|
||||
/// a <c>SubscribeAsync</c> landing mid-teardown cannot create a pump on the dying subscriber; a
|
||||
/// gateway close that never answers cannot park <c>ShutdownAsync</c>; a re-subscribe that
|
||||
/// straddles a teardown never rebinds into the next runtime's registry.
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverFixRound1Tests
|
||||
{
|
||||
internal const int WaitMs = 10_000;
|
||||
|
||||
internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("FixRound1"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
// ───────────────────────────── fakes ─────────────────────────────
|
||||
|
||||
private sealed class ScriptedPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public Func<int, IReadOnlyList<string>> Script { get; init; } = _ => ["Plat01"];
|
||||
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
=> Task.FromResult(Script(Interlocked.Increment(ref _calls)));
|
||||
}
|
||||
|
||||
internal sealed class ParkableSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle;
|
||||
private int _subscribeCalls;
|
||||
|
||||
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public List<string> Subscribed { get; } = [];
|
||||
public int SubscribeCalls => Volatile.Read(ref _subscribeCalls);
|
||||
|
||||
/// <summary>When set, UnsubscribeBulk parks until <see cref="ReleaseUnsubscribe"/>.</summary>
|
||||
public bool ParkUnsubscribe { get; set; }
|
||||
public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult();
|
||||
|
||||
/// <summary>When > 0, that SubscribeBulk call (1-based) parks until <see cref="ReleaseSubscribe"/>.</summary>
|
||||
public int ParkSubscribeCall { get; set; }
|
||||
public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult();
|
||||
|
||||
public async Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var call = Interlocked.Increment(ref _subscribeCalls);
|
||||
if (call == ParkSubscribeCall)
|
||||
{
|
||||
SubscribeEntered.TrySetResult();
|
||||
await _releaseSubscribe.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
var results = new List<SubscribeResult>();
|
||||
lock (Subscribed)
|
||||
{
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = Interlocked.Increment(ref _nextHandle);
|
||||
Subscribed.Add(r);
|
||||
HandleByRef[r] = h;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>Every handle passed to UnsubscribeBulk, in call order.</summary>
|
||||
public List<int> Unsubscribed { get; } = [];
|
||||
|
||||
public async Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
|
||||
if (ParkUnsubscribe)
|
||||
{
|
||||
UnsubscribeEntered.TrySetResult();
|
||||
await _releaseUnsubscribe.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { BoolValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource
|
||||
{
|
||||
public Channel<DeployEvent> Events { get; } = Channel.CreateUnbounded<DeployEvent>();
|
||||
public int Opened;
|
||||
|
||||
public async IAsyncEnumerable<DeployEvent> WatchAsync(
|
||||
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref Opened);
|
||||
await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent
|
||||
{
|
||||
Sequence = sequence,
|
||||
TimeOfLastDeployPresent = true,
|
||||
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
internal static HostState? StateOf(GalaxyDriver driver, string host) =>
|
||||
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
|
||||
|
||||
// ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ─────────────────────────────
|
||||
|
||||
/// <summary>A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses.</summary>
|
||||
[Fact]
|
||||
public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new ScriptedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
// The seed runs in the background (fix round 2, N1) — init does not wait on it.
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the probe watcher must be seeded after init");
|
||||
platforms.Calls.ShouldBe(1);
|
||||
lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
|
||||
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
|
||||
"a ScanState event must surface the seeded platform on GetHostStatuses");
|
||||
}
|
||||
|
||||
/// <summary>A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds.</summary>
|
||||
[Fact]
|
||||
public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var deploys = new ChannelDeployWatchSource();
|
||||
var platforms = new ScriptedPlatformSource
|
||||
{
|
||||
Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"],
|
||||
};
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
|
||||
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
|
||||
var raised = 0;
|
||||
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init");
|
||||
await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run");
|
||||
sub.Subscribed.ShouldBeEmpty();
|
||||
|
||||
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
|
||||
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed)
|
||||
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy
|
||||
|
||||
await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery");
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
|
||||
"the rediscovery must re-seed the probe set");
|
||||
platforms.Calls.ShouldBe(2);
|
||||
sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true);
|
||||
}
|
||||
|
||||
private sealed class ParkingPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public int ParkCall { get; init; }
|
||||
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Release() => _release.TrySetResult();
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var call = Interlocked.Increment(ref _calls);
|
||||
if (call == ParkCall)
|
||||
{
|
||||
Entered.TrySetResult();
|
||||
await _release.Task.WaitAsync(cancellationToken);
|
||||
return ["Plat01"]; // the pre-deploy view this pass fetched
|
||||
}
|
||||
return call == 1 ? [] : ["Plat01", "Eng02"];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts
|
||||
/// after it — otherwise the probe set would stay on the stale view until the next deploy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var deploys = new ChannelDeployWatchSource();
|
||||
var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null,
|
||||
subscriber: sub, platformSource: platforms, deployWatchSource: deploys);
|
||||
var raised = 0;
|
||||
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
|
||||
try
|
||||
{
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open");
|
||||
await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap
|
||||
await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
|
||||
(await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task,
|
||||
"the first rediscovery pass must be parked mid-fetch");
|
||||
|
||||
await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero));
|
||||
await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery");
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed");
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2,
|
||||
"the final probe set must reflect the latest deploy, not the parked pass's stale view");
|
||||
}
|
||||
|
||||
/// <summary>The production path (owned runtime) queries the platform source at init, with no DiscoverAsync.</summary>
|
||||
[Fact]
|
||||
public async Task Production_path_queries_the_platform_source_at_init()
|
||||
{
|
||||
var platforms = new ScriptedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms)
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => Task.CompletedTask,
|
||||
};
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init");
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
// ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before
|
||||
/// the subscriber is released) is refused and leaves no pump; before the shutdown the same
|
||||
/// call works (positive control).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the (background) seed must give the probe watcher its handle first");
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
|
||||
|
||||
sub.ParkUnsubscribe = true;
|
||||
var shutdown = driver.ShutdownAsync(CancellationToken.None);
|
||||
var subscribeCallsBefore = sub.SubscribeCalls;
|
||||
try
|
||||
{
|
||||
(await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task,
|
||||
"the teardown must be parked inside the probe watcher's unsubscribe");
|
||||
shutdown.IsCompleted.ShouldBeFalse();
|
||||
|
||||
// The pump is still live here (the teardown takes it after the probe watcher) — reusing it
|
||||
// is exactly the race: the call must be refused, not served on the dying subscriber.
|
||||
driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken");
|
||||
await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None));
|
||||
sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway");
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs.
|
||||
sub.ReleaseUnsubscribe();
|
||||
}
|
||||
await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown");
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
|
||||
}
|
||||
|
||||
// ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
|
||||
|
||||
/// <summary>A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline.</summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers()
|
||||
{
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource())
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => Task.CompletedTask,
|
||||
SessionCloseDeadline = TimeSpan.FromMilliseconds(300),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var closeToken = CancellationToken.None;
|
||||
driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct =>
|
||||
{
|
||||
closeToken = ct;
|
||||
closeEntered.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway
|
||||
};
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
sw.Stop();
|
||||
|
||||
closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted");
|
||||
closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s");
|
||||
driver.OwnedMxSessionForTests.ShouldBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
|
||||
}
|
||||
|
||||
// ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared
|
||||
/// registry; the same replay without a teardown rebinds (positive control).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind()
|
||||
{
|
||||
// Positive control first.
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription");
|
||||
sub.SubscribeCalls.ShouldBe(2);
|
||||
}
|
||||
|
||||
{
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
|
||||
var replay = driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
(await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task,
|
||||
"the replay must be parked inside its SubscribeBulk");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
|
||||
sub.ReleaseSubscribe();
|
||||
await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles");
|
||||
}
|
||||
}
|
||||
}
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Lane F fix round 2 (re-review of #735):
|
||||
/// <list type="bullet">
|
||||
/// <item>N1 — the init platform seed runs in the background: a slow or cancelled repository
|
||||
/// fetch can neither delay nor fail <c>InitializeAsync</c> (Galaxy's Tier-A Initialize budget is
|
||||
/// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown
|
||||
/// is discarded.</item>
|
||||
/// <item>N2 — a seed requested under a new runtime always gets a pass of its own, even when a
|
||||
/// worker from the previous runtime is still running.</item>
|
||||
/// <item>N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails
|
||||
/// and unsubscribes instead of registering into the next runtime's registry.</item>
|
||||
/// <item>The probe watcher's own subscribe re-checks its stop after the await.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverFixRound2Tests
|
||||
{
|
||||
private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>A platform source whose fetches park until released (optionally ignoring cancellation).</summary>
|
||||
private sealed class GatedPlatformSource : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public int ParkCall { get; init; } = 1;
|
||||
public bool IgnoreCancellation { get; init; }
|
||||
public Func<int, IReadOnlyList<string>> Result { get; init; } = _ => ["Plat01"];
|
||||
public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource<CancellationToken> ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Release() => _release.TrySetResult();
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var call = Interlocked.Increment(ref _calls);
|
||||
if (call == ParkCall)
|
||||
{
|
||||
ParkedToken.TrySetResult(cancellationToken);
|
||||
Entered.TrySetResult();
|
||||
if (IgnoreCancellation) await _release.Task;
|
||||
else await _release.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
return Result(call);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
return Task.FromException<IReadOnlyList<string>>(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Entered(TaskCompletionSource tcs, string because) =>
|
||||
(await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because);
|
||||
|
||||
// ───────────────────────────── N1: background init seed ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy
|
||||
/// inside its budget, and the teardown cancels the parked fetch.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
try
|
||||
{
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
sw.Stop();
|
||||
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
await Entered(platforms.Entered, "positive control: the seed fetch was started");
|
||||
sw.Elapsed.ShouldBeLessThan(InitBudget);
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
var token = await platforms.ParkedToken.Task;
|
||||
token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight");
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A fetch that is cancelled (the gateway's own deadline) does not fail init.</summary>
|
||||
[Fact]
|
||||
public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline"));
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
|
||||
|
||||
// The driver keeps serving after the failed seed.
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The production failure mode: the host's Tier-A Initialize timeout cancels the init token
|
||||
/// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and
|
||||
/// the seed — tied to the runtime, not to the init call — must survive that cancellation and
|
||||
/// still land.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
using var initCts = new CancellationTokenSource();
|
||||
try
|
||||
{
|
||||
initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk
|
||||
await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
|
||||
await Entered(platforms.Entered, "the seed fetch must be in flight");
|
||||
await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked");
|
||||
(await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse(
|
||||
"the seed belongs to the runtime; the init call's timeout must not cancel it");
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the seed must still land after the init token was cancelled");
|
||||
}
|
||||
|
||||
/// <summary>Positive control: a fetch that completes after init still populates host statuses.</summary>
|
||||
[Fact]
|
||||
public async Task A_fetch_completing_after_init_populates_host_statuses()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
try
|
||||
{
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
await Entered(platforms.Entered, "the seed must be parked");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the late seed must reach the probe watcher");
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump
|
||||
await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
|
||||
"a ScanState event must surface the late-seeded platform");
|
||||
}
|
||||
|
||||
// ───────────────────────────── N1 generation + N2: straddling worker ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A seed from the previous runtime that completes after a teardown + re-init is discarded,
|
||||
/// and the new runtime's own seed request still gets its own pass.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded()
|
||||
{
|
||||
var sub = new ParkableSubscriber();
|
||||
var platforms = new GatedPlatformSource
|
||||
{
|
||||
IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it
|
||||
Result = call => call == 1 ? ["Stale01"] : ["Plat01"],
|
||||
};
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms);
|
||||
try
|
||||
{
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
await Entered(platforms.Entered, "runtime 1's seed must be parked");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget);
|
||||
}
|
||||
finally
|
||||
{
|
||||
platforms.Release();
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass");
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"the new runtime must be seeded");
|
||||
driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]);
|
||||
lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded");
|
||||
}
|
||||
|
||||
// ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry
|
||||
/// back into the registry, and must release the handles it just acquired. Positive control:
|
||||
/// the same sync without a stop registers.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry()
|
||||
{
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control");
|
||||
}
|
||||
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
|
||||
var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry);
|
||||
|
||||
var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None);
|
||||
await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk");
|
||||
await watcher.StopAsync(CancellationToken.None);
|
||||
|
||||
sub.ReleaseSubscribe();
|
||||
await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register");
|
||||
watcher.WatchedPlatforms.ShouldBeEmpty();
|
||||
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"],
|
||||
"the handle acquired after the stop must be released");
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────────── N3: subscribe / read straddling a teardown ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing
|
||||
/// and unsubscribes its handles; a subscribe on the live runtime registers (positive control).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing()
|
||||
{
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 2 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers");
|
||||
|
||||
var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
try
|
||||
{
|
||||
await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk");
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sub.ReleaseSubscribe();
|
||||
}
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry");
|
||||
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry");
|
||||
}
|
||||
|
||||
/// <summary>The subscribe-once read path: same straddle, same outcome.</summary>
|
||||
[Fact]
|
||||
public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing()
|
||||
{
|
||||
var sub = new ParkableSubscriber { ParkSubscribeCall = 1 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None);
|
||||
try
|
||||
{
|
||||
await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk");
|
||||
await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
}
|
||||
finally
|
||||
{
|
||||
sub.ReleaseSubscribe();
|
||||
}
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)));
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]);
|
||||
}
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-4 / G-7 / G-11: the per-platform <c>ScanState</c> probing was structurally
|
||||
/// dead (its handles were never in the registry the pump resolves through, and a reconnect
|
||||
/// never re-subscribed them), discovery fetched the full galaxy twice, and
|
||||
/// <c>LastSuccessfulRead</c> never moved on data. These drive the real driver → pump → watcher
|
||||
/// → aggregator path with a fake subscriber and a fake hierarchy.
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverProbeWiringTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static GalaxyDriverOptions Opts() => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("ProbeWiring"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: false),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
private sealed class CountingHierarchySource : IGalaxyHierarchySource
|
||||
{
|
||||
public int Calls;
|
||||
|
||||
public Task<IReadOnlyList<GalaxyObject>> GetHierarchyAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref Calls);
|
||||
var platform = new GalaxyObject { GobjectId = 1, TagName = "Plat01", ContainedName = "Plat01" };
|
||||
platform.TemplateChain.Add("$WinPlatform");
|
||||
var tank = new GalaxyObject { GobjectId = 2, TagName = "Tank01", ContainedName = "Tank01", ParentGobjectId = 1 };
|
||||
tank.TemplateChain.Add("$UserDefined");
|
||||
tank.Attributes.Add(new GalaxyAttribute { AttributeName = "Level", FullTagReference = "Tank01.Level", MxDataType = 5 });
|
||||
return Task.FromResult<IReadOnlyList<GalaxyObject>>([platform, tank]);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StreamingSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle;
|
||||
|
||||
public List<string> Subscribed { get; } = [];
|
||||
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<SubscribeResult>();
|
||||
lock (Subscribed)
|
||||
{
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = Interlocked.Increment(ref _nextHandle);
|
||||
Subscribed.Add(r);
|
||||
HandleByRef[r] = h;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
}
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
||||
}
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { BoolValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
|
||||
public ValueTask EmitDoubleAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class NullBuilder : IAddressSpaceBuilder
|
||||
{
|
||||
public IAddressSpaceBuilder Folder(string browseName, string displayName) => this;
|
||||
public IVariableHandle Variable(string browseName, string displayName, DriverAttributeInfo attributeInfo) => new Handle(attributeInfo.FullName);
|
||||
public void AddProperty(string browseName, DriverDataType dataType, object? value) { }
|
||||
private sealed class Handle(string fullReference) : IVariableHandle
|
||||
{
|
||||
public string FullReference { get; } = fullReference;
|
||||
public IAlarmConditionSink MarkAsAlarmCondition(AlarmConditionInfo alarmInfo) => new NoopSink();
|
||||
}
|
||||
private sealed class NoopSink : IAlarmConditionSink
|
||||
{
|
||||
public void OnTransition(AlarmEventArgs args) { }
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
private static HostState? StateOf(GalaxyDriver driver, string host) =>
|
||||
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
|
||||
|
||||
/// <summary>
|
||||
/// A ScanState event on the probe handle updates GetHostStatuses; after a reconnect replay
|
||||
/// the probe is re-subscribed and events on the NEW handle keep updating it, while the dead
|
||||
/// pre-reconnect handle no longer does. Probe traffic never reaches public OnDataChange.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ScanState_event_updates_GetHostStatuses_and_survives_a_reconnect()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
var source = new CountingHierarchySource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
|
||||
var publicRefs = new List<string>();
|
||||
driver.OnDataChange += (_, e) => { lock (publicRefs) publicRefs.Add(e.FullReference); };
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
|
||||
// The pump starts on the first value subscription (the probe watcher does not start it).
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
|
||||
var probeHandle = sub.HandleByRef["Plat01.ScanState"];
|
||||
await sub.EmitBoolAsync(probeHandle, true);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running,
|
||||
"a ScanState=true event must surface Plat01 as Running");
|
||||
|
||||
// Reconnect replay: every pre-reconnect handle is dead; the probe must rebind.
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
var rebound = sub.HandleByRef["Plat01.ScanState"];
|
||||
rebound.ShouldNotBe(probeHandle, "replay must re-subscribe the probe and record the fresh handle");
|
||||
|
||||
await sub.EmitBoolAsync(rebound, false);
|
||||
await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Stopped,
|
||||
"a ScanState=false event on the post-reconnect handle must surface Plat01 as Stopped");
|
||||
|
||||
// The dead handle is no longer routed (positive control: the live one still is).
|
||||
await sub.EmitBoolAsync(probeHandle, true);
|
||||
await sub.EmitDoubleAsync(sub.HandleByRef["Tank01.Level"], 1.0);
|
||||
await WaitUntilAsync(() => { lock (publicRefs) return publicRefs.Contains("Tank01.Level"); }, "value traffic must still flow");
|
||||
StateOf(driver, "Plat01").ShouldBe(HostState.Stopped, "an event on the pre-reconnect handle must be dropped");
|
||||
|
||||
lock (publicRefs) publicRefs.ShouldNotContain("Plat01.ScanState", "probe traffic is the watcher's, never the host's");
|
||||
}
|
||||
|
||||
/// <summary>Discovery asks the gateway for the hierarchy once; the probe set is selected from that same sweep.</summary>
|
||||
[Fact]
|
||||
public async Task DiscoverAsync_fetches_the_hierarchy_once_and_still_syncs_the_probe_set()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
var source = new CountingHierarchySource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None);
|
||||
|
||||
source.Calls.ShouldBe(1, "the probe selection must reuse the discoverer's sweep");
|
||||
sub.Subscribed.ShouldContain("Plat01.ScanState", "positive control: the platform probe was still synced");
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]);
|
||||
}
|
||||
|
||||
/// <summary>Inbound stream data advances LastSuccessfulRead (throttled); the throttle holds the stamp still.</summary>
|
||||
[Fact]
|
||||
public async Task Stream_data_advances_LastSuccessfulRead_under_a_throttle()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
||||
{
|
||||
LivenessStampThrottle = TimeSpan.FromHours(1),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var initStamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
await Task.Delay(20);
|
||||
|
||||
var delivered = 0;
|
||||
driver.OnDataChange += (_, _) => Interlocked.Increment(ref delivered);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
var handle = sub.HandleByRef["Tank01.Level"];
|
||||
|
||||
await sub.EmitDoubleAsync(handle, 1.0);
|
||||
await WaitUntilAsync(() => Volatile.Read(ref delivered) == 1, "first event must be delivered");
|
||||
var first = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
first.ShouldBeGreaterThan(initStamp, "data on the stream is liveness evidence");
|
||||
|
||||
await Task.Delay(20);
|
||||
await sub.EmitDoubleAsync(handle, 2.0);
|
||||
await WaitUntilAsync(() => Volatile.Read(ref delivered) == 2, "second event must be delivered");
|
||||
driver.GetHealth().LastSuccessfulRead.ShouldBe(first, "inside the throttle window the stamp holds still");
|
||||
}
|
||||
}
|
||||
+386
@@ -0,0 +1,386 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="GalaxyDriver.ShutdownAsync"/> used to be a no-op (review 2026-10-21, G-1/G-2/G-9):
|
||||
/// the gRPC runtime outlived the driver on every respawn, removal and <c>/raw</c> browse, and
|
||||
/// the host's comms-loss recovery (<c>ShutdownAsync</c> → <c>InitializeAsync</c> on the SAME
|
||||
/// object) neither tore down nor rebuilt anything. These tests pin the teardown at every
|
||||
/// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed,
|
||||
/// the owned channels + session — and the rebuild on the next <c>InitializeAsync</c>.
|
||||
/// </summary>
|
||||
public sealed class GalaxyDriverShutdownTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("ShutdownTest"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
// ───────────────────────────── fakes ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Subscriber whose event stream records the token it was opened with and completes a
|
||||
/// signal when that token fires — the observable for "the StreamEvents stream was cancelled".
|
||||
/// </summary>
|
||||
private sealed class StreamingSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle;
|
||||
|
||||
public Dictionary<string, int> HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public List<int> Unsubscribed { get; } = [];
|
||||
public int StreamsOpened;
|
||||
public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<SubscribeResult>();
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = Interlocked.Increment(ref _nextHandle);
|
||||
HandleByRef[r] = h;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
||||
}
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (Unsubscribed) Unsubscribed.AddRange(itemHandles);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref StreamsOpened);
|
||||
using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult());
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when
|
||||
/// the enumerator is cancelled or completes) — the observable for "exactly one watcher".
|
||||
/// </summary>
|
||||
private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource
|
||||
{
|
||||
private int _open;
|
||||
private int _opened;
|
||||
public Channel<DeployEvent> Current { get; private set; } = Channel.CreateUnbounded<DeployEvent>();
|
||||
|
||||
public int Open => Volatile.Read(ref _open);
|
||||
public int Opened => Volatile.Read(ref _opened);
|
||||
|
||||
public async IAsyncEnumerable<DeployEvent> WatchAsync(
|
||||
DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _open);
|
||||
Interlocked.Increment(ref _opened);
|
||||
var channel = Current = Channel.CreateUnbounded<DeployEvent>();
|
||||
try
|
||||
{
|
||||
await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref _open);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent
|
||||
{
|
||||
Sequence = sequence,
|
||||
TimeOfLastDeployPresent = true,
|
||||
TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime),
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Canned platform set so a production-path init never dials the (fake) gateway for one.</summary>
|
||||
private sealed class NoPlatforms : IGalaxyPlatformSource
|
||||
{
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<string>>([]);
|
||||
}
|
||||
|
||||
private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed
|
||||
{
|
||||
public int Starts;
|
||||
public int Disposes;
|
||||
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
|
||||
public void Start() => Starts++;
|
||||
public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition(
|
||||
"Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise,
|
||||
AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi"));
|
||||
public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; }
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────────── G-1: streams + pump ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the
|
||||
/// registry is empty. Positive control: an event delivered through that same stream
|
||||
/// reached OnDataChange before the shutdown.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
driver.OnDataChange += (_, e) => delivered.TrySetResult(e);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeTrue();
|
||||
|
||||
await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0);
|
||||
(await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task,
|
||||
"positive control: the stream must deliver before the shutdown");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
// Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the
|
||||
// enumerator's cancellation registration has run by the time it returns (observed flaky).
|
||||
(await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task,
|
||||
"ShutdownAsync must cancel the StreamEvents stream");
|
||||
driver.HasEventPumpForTests.ShouldBeFalse();
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init");
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
|
||||
driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT
|
||||
/// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive
|
||||
/// control on both sides of the shutdown.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it()
|
||||
{
|
||||
var feed = new RecordingAlarmFeed();
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
|
||||
subscriber: sub, alarmFeed: feed);
|
||||
var received = 0;
|
||||
driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
|
||||
feed.Emit();
|
||||
received.ShouldBe(1, "positive control: the feed is wired before the shutdown");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
feed.Emit();
|
||||
received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent");
|
||||
feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector");
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None);
|
||||
feed.Emit();
|
||||
received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed");
|
||||
feed.Starts.ShouldBe(2);
|
||||
}
|
||||
|
||||
// ───────────────────────────── G-2: one DeployWatcher ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch
|
||||
/// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly
|
||||
/// once — five stacked watchers would raise five times.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher()
|
||||
{
|
||||
var source = new CountingDeployWatchSource();
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
deployWatchSource: source);
|
||||
var raised = 0;
|
||||
driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
for (var cycle = 0; cycle < 5; cycle++)
|
||||
{
|
||||
// Let this cycle's watch stream actually open before the next shutdown, so every cycle
|
||||
// holds a live stream the teardown has to cancel (a loop cancelled before its first
|
||||
// open would make the count vacuous).
|
||||
await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open");
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
}
|
||||
|
||||
await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1,
|
||||
$"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}");
|
||||
driver.DeployWatcherForTests.ShouldNotBeNull();
|
||||
|
||||
// Bootstrap (suppressed) then a genuine change.
|
||||
await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
|
||||
await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero));
|
||||
await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery");
|
||||
await Task.Delay(200);
|
||||
Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy");
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream");
|
||||
driver.DeployWatcherForTests.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one.</summary>
|
||||
[Fact]
|
||||
public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher()
|
||||
{
|
||||
var source = new CountingDeployWatchSource();
|
||||
var sub = new StreamingSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
deployWatchSource: source);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var first = driver.DeployWatcherForTests;
|
||||
await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open");
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1,
|
||||
$"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}");
|
||||
driver.DeployWatcherForTests.ShouldNotBeSameAs(first);
|
||||
}
|
||||
|
||||
// ───────────────────────────── G-1/G-9: owned runtime, channels + session ─────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// The production path (no injected seams): InitializeAsync builds the owned client, session
|
||||
/// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the
|
||||
/// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's
|
||||
/// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC
|
||||
/// channels are real but lazy (nothing dials until a call is made).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds()
|
||||
{
|
||||
var opens = 0;
|
||||
var source = new CountingDeployWatchSource();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source,
|
||||
platformSource: new NoPlatforms())
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; },
|
||||
};
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
opens.ShouldBe(1);
|
||||
var client = driver.OwnedMxClientForTests.ShouldNotBeNull();
|
||||
var session = driver.OwnedMxSessionForTests.ShouldNotBeNull();
|
||||
session.IsConnected.ShouldBeTrue();
|
||||
driver.SupervisorForTests.ShouldNotBeNull();
|
||||
driver.ProbeWatcherForTests.ShouldNotBeNull();
|
||||
driver.DeployWatcherForTests.ShouldNotBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session");
|
||||
await Should.ThrowAsync<ObjectDisposedException>(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None),
|
||||
"the owned gateway channel must be disposed");
|
||||
driver.OwnedMxClientForTests.ShouldBeNull();
|
||||
driver.OwnedMxSessionForTests.ShouldBeNull();
|
||||
driver.SupervisorForTests.ShouldBeNull();
|
||||
driver.ProbeWatcherForTests.ShouldBeNull();
|
||||
driver.DeployWatcherForTests.ShouldBeNull();
|
||||
await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled");
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Unknown);
|
||||
driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp);
|
||||
|
||||
// The watchdog's second half: a fresh runtime on the same object.
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one");
|
||||
driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client);
|
||||
driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session);
|
||||
driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue();
|
||||
driver.SupervisorForTests.ShouldNotBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A connect that fails mid-build releases the partial runtime (no orphaned client per
|
||||
/// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds()
|
||||
{
|
||||
var attempts = 0;
|
||||
await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null,
|
||||
platformSource: new NoPlatforms())
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => ++attempts == 1
|
||||
? throw new InvalidOperationException("gateway refused")
|
||||
: Task.CompletedTask,
|
||||
};
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(() => driver.InitializeAsync("{}", CancellationToken.None));
|
||||
ex.Message.ShouldBe("gateway refused");
|
||||
driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client");
|
||||
driver.OwnedMxSessionForTests.ShouldBeNull();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Faulted);
|
||||
driver.GetHealth().LastError.ShouldBe("gateway refused");
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
attempts.ShouldBe(2);
|
||||
driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue();
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
/// <summary>DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op.</summary>
|
||||
[Fact]
|
||||
public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless()
|
||||
{
|
||||
var sub = new StreamingSubscriber();
|
||||
var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
await Should.NotThrowAsync(async () => await driver.DisposeAsync());
|
||||
await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None));
|
||||
await Should.ThrowAsync<ObjectDisposedException>(() => driver.InitializeAsync("{}", CancellationToken.None));
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="GalaxyMxSession.CloseAsync"/> bounds the gateway <c>CloseSession</c> round-trip by
|
||||
/// the caller's token (the SDK dispose closes under its 30 s call timeout) and leaves the session
|
||||
/// re-openable.
|
||||
/// </summary>
|
||||
public sealed class GalaxyMxSessionCloseTests
|
||||
{
|
||||
/// <summary>A close against a never-answering gateway returns when the token fires, without throwing.</summary>
|
||||
[Fact]
|
||||
public async Task CloseAsync_returns_at_the_token_deadline_against_a_never_answering_gateway()
|
||||
{
|
||||
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = ct => Task.Delay(Timeout.Infinite, ct),
|
||||
};
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
session.IsConnected.ShouldBeTrue();
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
await session.CloseAsync(cts.Token).WaitAsync(TimeSpan.FromSeconds(10));
|
||||
sw.Stop();
|
||||
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5));
|
||||
session.IsConnected.ShouldBeFalse();
|
||||
|
||||
// Still usable: a close is not a dispose.
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
session.IsConnected.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>Positive control: a close that answers completes and reports the token it was handed.</summary>
|
||||
[Fact]
|
||||
public async Task CloseAsync_completes_when_the_gateway_answers()
|
||||
{
|
||||
CancellationToken seen = default;
|
||||
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = ct => { seen = ct; return Task.CompletedTask; },
|
||||
};
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
await session.CloseAsync(cts.Token);
|
||||
|
||||
seen.ShouldBe(cts.Token, "the caller's token must reach the gateway round-trip");
|
||||
session.IsConnected.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The driver's teardown is CloseAsync (bounded) then DisposeAsync (final). The dispose must
|
||||
/// not issue a second gateway close; a fresh connect after a close must close again (positive
|
||||
/// control: the guard keys on "was connected", not on "ever closed").
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CloseAsync_then_DisposeAsync_closes_the_gateway_session_once()
|
||||
{
|
||||
var closes = 0;
|
||||
var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
|
||||
};
|
||||
await session.ConnectAsync(null!, CancellationToken.None);
|
||||
|
||||
await session.CloseAsync(CancellationToken.None);
|
||||
await session.DisposeAsync();
|
||||
closes.ShouldBe(1, "dispose after a close must not close the gateway session again");
|
||||
|
||||
var reopened = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest"))
|
||||
{
|
||||
OpenAndRegisterOverrideForTests = _ => Task.CompletedTask,
|
||||
CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; },
|
||||
};
|
||||
await reopened.ConnectAsync(null!, CancellationToken.None);
|
||||
await reopened.CloseAsync(CancellationToken.None);
|
||||
await reopened.ConnectAsync(null!, CancellationToken.None);
|
||||
await reopened.DisposeAsync();
|
||||
closes.ShouldBe(3, "each live session closes exactly once");
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
/// <see cref="ActivityListener"/> rather than asserting on internal state, so the
|
||||
/// tests double as documentation of the listener-side contract.
|
||||
/// </summary>
|
||||
[Collection(ProcessWideTelemetryCollection.Name)]
|
||||
public sealed class GalaxyTelemetryTests
|
||||
{
|
||||
/// <summary>Subscribes an ActivityListener for the test, captures each spawned activity.</summary>
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-8: a batch of writes ran strictly serially, three round-trips per
|
||||
/// first-touch tag. Distinct tags now run as concurrent lanes bounded at
|
||||
/// <see cref="GatewayGalaxyDataWriter.MaxConcurrentWriteLanes"/>; same-tag writes keep their
|
||||
/// order inside one lane; results stay at their request's index.
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyDataWriterConcurrencyTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static GatewayGalaxyDataWriter NewWriter()
|
||||
=> new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0);
|
||||
|
||||
/// <summary>
|
||||
/// Fake ops whose AddItem parks every caller until <see cref="ReleaseAt"/> callers are in
|
||||
/// flight at once (or a safety delay elapses), recording the peak concurrency.
|
||||
/// </summary>
|
||||
private sealed class GatingOps : IMxWriteOps
|
||||
{
|
||||
private int _inFlight;
|
||||
private int _peak;
|
||||
private int _nextHandle;
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly ConcurrentDictionary<string, int> _handles = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public int ReleaseAt { get; init; } = int.MaxValue;
|
||||
public int Peak => Volatile.Read(ref _peak);
|
||||
public ConcurrentQueue<(int Handle, MxValue Value)> RawWrites { get; } = new();
|
||||
public ConcurrentQueue<string> AddItems { get; } = new();
|
||||
public Func<int, MxCommandReply>? WriteResponder { get; init; }
|
||||
|
||||
/// <summary>Deterministic handle per tag when set; otherwise handles are assigned in AddItem completion order.</summary>
|
||||
public Func<string, int>? HandleFor { get; init; }
|
||||
|
||||
public string SessionId => "sess";
|
||||
public int ServerHandle => 1;
|
||||
|
||||
public async Task<int> AddItemAsync(string fullRef, CancellationToken ct)
|
||||
{
|
||||
AddItems.Enqueue(fullRef);
|
||||
var now = Interlocked.Increment(ref _inFlight);
|
||||
int seen;
|
||||
do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen);
|
||||
if (now >= ReleaseAt) _release.TrySetResult();
|
||||
// Park on the signal; the delay is only a safety net against a wedged test (a starved
|
||||
// pool must not release the gate early and turn the peak assertion flaky).
|
||||
await Task.WhenAny(_release.Task, Task.Delay(WaitMs, ct));
|
||||
Interlocked.Decrement(ref _inFlight);
|
||||
return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle));
|
||||
}
|
||||
|
||||
public Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken ct)
|
||||
=> Task.FromResult(new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } });
|
||||
|
||||
public Task<MxCommandReply> WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct)
|
||||
{
|
||||
RawWrites.Enqueue((itemHandle, value));
|
||||
return Task.FromResult(WriteResponder?.Invoke(itemHandle) ?? GoodStatusRow());
|
||||
}
|
||||
|
||||
// A reply carrying a Good MX status ROW (not an empty statuses array): the writer's static
|
||||
// meters are process-wide, and an empty-statuses Good would increment galaxy.writes.unconfirmed
|
||||
// under the meter tests running in parallel in another class.
|
||||
private static MxCommandReply GoodStatusRow()
|
||||
{
|
||||
var reply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } };
|
||||
reply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok });
|
||||
return reply;
|
||||
}
|
||||
}
|
||||
|
||||
private static SecurityClassification Free(string _) => SecurityClassification.FreeAccess;
|
||||
|
||||
/// <summary>Sixteen distinct tags: lanes overlap (not serial) and never exceed the cap of eight.</summary>
|
||||
[Fact]
|
||||
public async Task Distinct_tags_write_concurrently_bounded_at_the_lane_cap()
|
||||
{
|
||||
var ops = new GatingOps { ReleaseAt = GatewayGalaxyDataWriter.MaxConcurrentWriteLanes };
|
||||
var writes = Enumerable.Range(0, 16).Select(i => new WriteRequest($"Tag{i}.PV", (double)i)).ToArray();
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
ops.Peak.ShouldBe(GatewayGalaxyDataWriter.MaxConcurrentWriteLanes,
|
||||
"first-touch lanes must overlap (serial = 1) and stay bounded (unbounded > 8)");
|
||||
results.Count.ShouldBe(16);
|
||||
results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
|
||||
ops.AddItems.Count.ShouldBe(16, "one AddItem per distinct tag");
|
||||
ops.RawWrites.Count.ShouldBe(16);
|
||||
}
|
||||
|
||||
/// <summary>Writes to the same tag stay in input order inside their lane and share one AddItem.</summary>
|
||||
[Fact]
|
||||
public async Task Same_tag_writes_stay_ordered_and_share_one_handle()
|
||||
{
|
||||
var ops = new GatingOps { ReleaseAt = 2 };
|
||||
var writes = new[]
|
||||
{
|
||||
new WriteRequest("Tank01.SP", 1.0),
|
||||
new WriteRequest("Tank02.SP", 10.0),
|
||||
new WriteRequest("Tank01.SP", 2.0),
|
||||
new WriteRequest("Tank01.SP", 3.0),
|
||||
};
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
results.Count.ShouldBe(4);
|
||||
results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good);
|
||||
ops.AddItems.Count(a => a == "Tank01.SP").ShouldBe(1, "a lane AddItems its tag once");
|
||||
var tank01 = ops.RawWrites.Where(w => w.Handle == ops.RawWrites.First(x => x.Value.DoubleValue == 1.0).Handle)
|
||||
.Select(w => w.Value.DoubleValue).ToArray();
|
||||
tank01.ShouldBe([1.0, 2.0, 3.0], "same-tag writes must keep their authored order");
|
||||
}
|
||||
|
||||
/// <summary>A per-tag failure lands at that request's index while the other lanes stay Good.</summary>
|
||||
[Fact]
|
||||
public async Task Results_land_at_their_request_index()
|
||||
{
|
||||
var ops = new GatingOps
|
||||
{
|
||||
ReleaseAt = 3,
|
||||
HandleFor = tag => tag[0] - 'A' + 1, // A.PV → 1, B.PV → 2, C.PV → 3
|
||||
WriteResponder = handle => new MxCommandReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus
|
||||
{
|
||||
Code = handle == 2 ? ProtocolStatusCode.MxaccessFailure : ProtocolStatusCode.Ok,
|
||||
},
|
||||
},
|
||||
};
|
||||
var writes = new[]
|
||||
{
|
||||
new WriteRequest("A.PV", 1.0),
|
||||
new WriteRequest("B.PV", 2.0), // refused by the gateway
|
||||
new WriteRequest("C.PV", 3.0),
|
||||
};
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
results.Select(r => r.StatusCode).ShouldBe(
|
||||
[StatusCodeMap.Good, StatusCodeMap.BadCommunicationError, StatusCodeMap.Good],
|
||||
"each result must sit at its request's index regardless of lane completion order");
|
||||
}
|
||||
|
||||
/// <summary>A single-tag batch takes the direct path and still produces one result per request.</summary>
|
||||
[Fact]
|
||||
public async Task Single_tag_batch_runs_serially_in_order()
|
||||
{
|
||||
var ops = new GatingOps { ReleaseAt = 1 };
|
||||
var writes = new[] { new WriteRequest("X.PV", 1.0), new WriteRequest("X.PV", 2.0) };
|
||||
|
||||
var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromMilliseconds(WaitMs));
|
||||
|
||||
results.Count.ShouldBe(2);
|
||||
ops.Peak.ShouldBe(1);
|
||||
ops.RawWrites.Select(w => w.Value.DoubleValue).ShouldBe([1.0, 2.0]);
|
||||
}
|
||||
}
|
||||
+1
@@ -13,6 +13,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
/// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal
|
||||
/// <see cref="IMxWriteOps"/> seam (the SDK session types are sealed + internal-ctor and cannot be faked).
|
||||
/// </summary>
|
||||
[Collection(ProcessWideTelemetryCollection.Name)]
|
||||
public sealed class GatewayGalaxyDataWriterFailClosedTests
|
||||
{
|
||||
private static GatewayGalaxyDataWriter NewWriter()
|
||||
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 G-3: <see cref="ReconnectSupervisor.ReportTransportFailure"/> allocated a
|
||||
/// new <see cref="CancellationTokenSource"/> per recovery cycle and only the last one was ever
|
||||
/// disposed. Each cycle now releases its own CTS when its loop exits, and
|
||||
/// <see cref="ReconnectSupervisor.StopAsync"/> awaits the loop instead of blocking on it.
|
||||
/// </summary>
|
||||
public sealed class ReconnectSupervisorCycleCtsTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static ReconnectOptions FastBackoff() =>
|
||||
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
|
||||
|
||||
/// <summary>Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control).</summary>
|
||||
[Fact]
|
||||
public async Task Every_completed_recovery_cycle_releases_its_own_cts()
|
||||
{
|
||||
var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var hold = new[] { false }; // read on the recovery loop's thread, written here — Volatile both ways
|
||||
using var sup = new ReconnectSupervisor(
|
||||
reopen: async _ =>
|
||||
{
|
||||
if (Volatile.Read(ref hold[0]))
|
||||
{
|
||||
reopenEntered.TrySetResult();
|
||||
await holdReopen.Task;
|
||||
}
|
||||
},
|
||||
replay: _ => Task.CompletedTask,
|
||||
options: FastBackoff());
|
||||
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
sup.ReportTransportFailure(new IOException($"drop {i}"));
|
||||
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
|
||||
}
|
||||
await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS");
|
||||
sup.CyclesStarted.ShouldBe(3);
|
||||
|
||||
// Positive control: a cycle parked inside reopen still holds its CTS.
|
||||
Volatile.Write(ref hold[0], true);
|
||||
sup.ReportTransportFailure(new IOException("drop 3"));
|
||||
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
|
||||
sup.CyclesStarted.ShouldBe(4);
|
||||
sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits");
|
||||
|
||||
holdReopen.TrySetResult();
|
||||
await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token);
|
||||
await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit");
|
||||
}
|
||||
|
||||
/// <summary>StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking.</summary>
|
||||
[Fact]
|
||||
public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts()
|
||||
{
|
||||
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var sup = new ReconnectSupervisor(
|
||||
reopen: async ct =>
|
||||
{
|
||||
reopenEntered.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
},
|
||||
replay: _ => Task.CompletedTask,
|
||||
options: FastBackoff());
|
||||
|
||||
sup.ReportTransportFailure(new IOException("drop"));
|
||||
(await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task);
|
||||
|
||||
var stop = sup.StopAsync();
|
||||
(await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return");
|
||||
sup.CyclesReleased.ShouldBe(1);
|
||||
await Should.NotThrowAsync(sup.StopAsync); // idempotent
|
||||
Should.NotThrow(sup.Dispose);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user