Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverReadTests.cs
T
Joseph Doherty 4fdc3708f4 test(galaxy): presence waits survive a starved/loaded runner; absence asserted behind positive controls (#858)
Under heavy parallel load (14 concurrent test processes + 14 CPU burners, with
DOTNET_ThreadPool_ForceMinWorkerThreads=1 as the reproducer) the Galaxy tests
with 1-3 s fixed waits failed 16/70 runs. Every failure was a background loop
started with Task.Run (alarm feed, DeployWatcher, EventPump, ReconnectSupervisor)
not being SCHEDULED within the ceiling - e.g. DeployWatcher's stream had not even
been opened after 2 s. No production race: the code under test was checked for
each (alarm feed dispatch is synchronous in-order, DeployWatcher/ReconnectSupervisor
state is lock- or loop-local, subscribe registers before returning).

- New shared TestWait (UntilAsync / ForAsync / HealthyAsync): 30 s presence
  ceiling, failure reports elapsed ms + an observed-state snapshot (#856 shape;
  GalaxyDriverFixRound1Tests.WaitUntilAsync now delegates to it).
- Converted the short waits in GatewayGalaxyAlarmFeedTests, DeployWatcherTests,
  GalaxyDriverSubscribeTests, GalaxyDriverRawPathKeyingTests (its WaitForAsync
  never threw on timeout), EventPumpStreamFaultTests, EventPumpReplayGapTests,
  ReconnectSupervisorTests; FixRound2 InitBudget 2 s -> 10 s (the parked fetch
  never completes, so any finite bound keeps the verdict).
- Absence by short sleep replaced with positive controls: DeployWatcher bootstrap
  and same-deploy-time (follow with a real change, assert exactly that one raise -
  verified red when suppression is removed), subscribe failed-tag / unsubscribed
  handle (follow with a live handle's event), faulted pump (wait for its
  onStreamFault instead of 100 ms).
- Fixed sleeps that raced scheduling: subscribe-once read tests wait for the
  read's registration (TrackedSubscriptionCountForTests) instead of 50 ms;
  supervisor Dispose test waits for reopen entry instead of 50 ms; the
  already-Healthy WaitForHealthy test asserts synchronous completion instead of a
  wall-clock bound. FakeSubscriber.Map is now a ConcurrentDictionary.

Loaded run (14 parallel x 5, 14 burners, ForceMinWorkerThreads=1):
before 16/70 failed runs; after 0/350 (70 + 210 + 70). Without the thread-pool
override: 0/70 before and after on this host. Serial: 472 passed, 8 skipped.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 10:18:22 -05:00

244 lines
12 KiB
C#

using ZB.MOM.WW.MxGateway.Contracts.Proto;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Tests for <see cref="GalaxyDriver"/>'s <c>IReadable</c> wiring. PR 4.2 ships the
/// abstraction (<see cref="IGalaxyDataReader"/>) and the wiring; PR 4.4 supplies the
/// production gateway-backed reader. These tests verify the wiring against a fake
/// reader plus the explicit "no reader → NotSupportedException" fallback that protects
/// deployments running on this PR from silently producing wrong reads.
/// </summary>
public sealed class GalaxyDriverReadTests
{
private static GalaxyDriverOptions Opts() => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("OtOpcUa-A"),
new GalaxyRepositoryOptions(),
new GalaxyReconnectOptions());
private sealed class FakeReader : IGalaxyDataReader
{
/// <summary>Gets the last read request.</summary>
public IReadOnlyList<string>? LastRequest { get; private set; }
/// <summary>Gets or sets the function that decides the result for a given tag list.</summary>
public Func<IReadOnlyList<string>, IReadOnlyList<DataValueSnapshot>> Decide { get; set; } =
tags => tags.Select(t => new DataValueSnapshot(
Value: t,
StatusCode: StatusCodeMap.Good,
SourceTimestampUtc: DateTime.UtcNow,
ServerTimestampUtc: DateTime.UtcNow)).ToArray();
/// <inheritdoc />
public Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(
IReadOnlyList<string> fullReferences, CancellationToken cancellationToken)
{
LastRequest = fullReferences;
return Task.FromResult(Decide(fullReferences));
}
}
/// <summary>Verifies that ReadAsync routes through the injected reader.</summary>
[Fact]
public async Task ReadAsync_RoutesThroughInjectedReader()
{
var reader = new FakeReader();
var driver = new GalaxyDriver("g", Opts(), hierarchySource: null, dataReader: reader);
var result = await driver.ReadAsync(["Tank1.Level", "Tank2.Level"], CancellationToken.None);
reader.LastRequest.ShouldBe(new[] { "Tank1.Level", "Tank2.Level" });
result.Count.ShouldBe(2);
result[0].Value.ShouldBe("Tank1.Level");
result[0].StatusCode.ShouldBe(StatusCodeMap.Good);
}
/// <summary>Verifies that ReadAsync returns empty without calling the reader for an empty request.</summary>
[Fact]
public async Task ReadAsync_EmptyRequest_ReturnsEmpty_WithoutCallingReader()
{
var reader = new FakeReader();
var driver = new GalaxyDriver("g", Opts(), hierarchySource: null, dataReader: reader);
var result = await driver.ReadAsync([], CancellationToken.None);
result.ShouldBeEmpty();
reader.LastRequest.ShouldBeNull();
}
/// <summary>Verifies that ReadAsync throws when seams and production runtime are not built.</summary>
[Fact]
public async Task ReadAsync_NoSeams_AndNoProductionRuntime_Throws()
{
// Construction without seams + without InitializeAsync gives a driver where
// _dataReader and _subscriber are both null. The follow-up read path can't
// synthesise a Read without one, so it surfaces a NotSupportedException
// pointing at the misuse rather than NullRef'ing inside the pump path.
var driver = new GalaxyDriver("g", Opts(), hierarchySource: null);
var ex = await Should.ThrowAsync<NotSupportedException>(() =>
driver.ReadAsync(["x"], CancellationToken.None));
ex.Message.ShouldContain("production runtime not built");
}
/// <summary>Verifies that ReadAsync throws after the driver is disposed.</summary>
[Fact]
public async Task ReadAsync_AfterDispose_Throws()
{
var driver = new GalaxyDriver("g", Opts(), hierarchySource: null, dataReader: new FakeReader());
driver.Dispose();
await Should.ThrowAsync<ObjectDisposedException>(() =>
driver.ReadAsync(["x"], CancellationToken.None));
}
/// <summary>Verifies that ReadAsync resolves from the first OnDataChange event on the subscribe-once path.</summary>
[Fact]
public async Task ReadAsync_SubscribeOncePath_ResolvesFromFirstOnDataChange()
{
// Follow-up #1: when no test reader is injected but a subscriber IS, the driver
// synthesises a Read by subscribing, waiting for the first OnDataChange event
// per item handle (gw pushes initial value), then unsubscribing.
var subscriber = new GalaxyDriverSubscribeTests.FakeSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
var readTask = driver.ReadAsync(["Tank.Level"], CancellationToken.None);
// Push the "initial value" event the gw would emit immediately after SubscribeBulk.
// Wait until the read's bindings are registered (#858: a fixed 50 ms beat was a timing guess — an
// event emitted before registration is dropped by the pump and the read then times out).
await TestWait.UntilAsync(() => driver.TrackedSubscriptionCountForTests == 1,
"the subscribe-once read must register its bindings");
var itemHandle = subscriber.Map["Tank.Level"];
await subscriber.EmitOnDataChangeAsync(itemHandle, 42.0);
var result = await readTask;
result.Count.ShouldBe(1);
result[0].Value.ShouldBe(42.0);
// Cleanup unsubscribed the live handle.
subscriber.UnsubscribedHandles.ShouldContain(itemHandle);
}
/// <summary>Verifies that ReadAsync surfaces rejected tags as bad status on the subscribe-once path.</summary>
[Fact]
public async Task ReadAsync_SubscribeOncePath_RejectedTagSurfacesAsBadStatus()
{
// gw rejects "Bad" at SubscribeBulk; the read path completes that slot with a
// Bad-status snapshot rather than waiting forever for an event that won't come.
var subscriber = new GalaxyDriverSubscribeTests.FakeSubscriber { Decide = tag => tag != "Bad" };
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
var readTask = driver.ReadAsync(["Good", "Bad"], CancellationToken.None);
await TestWait.UntilAsync(() => driver.TrackedSubscriptionCountForTests == 1,
"the subscribe-once read must register its bindings");
await subscriber.EmitOnDataChangeAsync(subscriber.Map["Good"], 1.0);
var result = await readTask;
result.Count.ShouldBe(2);
result[0].Value.ShouldBe(1.0);
result[1].StatusCode.ShouldBe(0x80000000u); // Bad
}
/// <summary>Verifies that ReadAsync preserves reader status codes.</summary>
[Fact]
public async Task ReadAsync_PreservesReaderStatusCodes()
{
var reader = new FakeReader
{
Decide = tags => new DataValueSnapshot[]
{
new(42.0, StatusCodeMap.Good, DateTime.UtcNow, DateTime.UtcNow),
new(null, StatusCodeMap.BadNotConnected, null, DateTime.UtcNow),
},
};
var driver = new GalaxyDriver("g", Opts(), hierarchySource: null, dataReader: reader);
var result = await driver.ReadAsync(["a", "b"], CancellationToken.None);
result[0].StatusCode.ShouldBe(StatusCodeMap.Good);
result[1].StatusCode.ShouldBe(StatusCodeMap.BadNotConnected);
}
/// <summary>A never-publishing tag with CancellationToken.None must complete with BadTimeout at the internal deadline (06/S-3).</summary>
[Fact]
public async Task ReadAsync_SubscribeOncePath_NeverPublishes_NoCallerToken_TimesOutWithBadTimeout()
{
var subscriber = new GalaxyDriverSubscribeTests.FakeSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
{
ReadViaSubscribeDeadline = TimeSpan.FromMilliseconds(200),
};
var readTask = driver.ReadAsync(["Silent.Tag"], CancellationToken.None);
var finished = await Task.WhenAny(readTask, Task.Delay(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken));
finished.ShouldBe(readTask);
var result = await readTask;
result.Count.ShouldBe(1);
result[0].StatusCode.ShouldBe(StatusCodeMap.BadTimeout);
subscriber.UnsubscribedHandles.ShouldContain(subscriber.Map["Silent.Tag"]);
}
/// <summary>
/// A subscribe RPC that outlives the internal deadline (caller passed None) must yield BadTimeout for
/// every tag, not an OperationCanceledException, and the handles the late-completing RPC created must
/// still be unsubscribed.
/// </summary>
[Fact]
public async Task ReadAsync_SubscribeOncePath_SlowSubscribeBulk_NoCallerToken_ReturnsBadTimeoutAndUnsubscribesLateHandles()
{
var subscriber = new SlowSubscribeSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
{
ReadViaSubscribeDeadline = TimeSpan.FromMilliseconds(200),
};
var result = await driver.ReadAsync(["Slow.A", "Slow.B"], CancellationToken.None)
.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
result.Count.ShouldBe(2);
result.ShouldAllBe(r => r.StatusCode == StatusCodeMap.BadTimeout);
// The RPC finally completes after the read gave up; its handles must not leak.
subscriber.Gate.SetResult();
var deadline = DateTime.UtcNow.AddSeconds(10);
while (subscriber.UnsubscribedHandles.Count < 2 && DateTime.UtcNow < deadline) await Task.Delay(20, TestContext.Current.CancellationToken);
subscriber.UnsubscribedHandles.OrderBy(h => h).ShouldBe([101, 102]);
}
private sealed class SlowSubscribeSubscriber : IGalaxySubscriber
{
private readonly List<int> _unsubscribed = [];
public TaskCompletionSource Gate { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public IReadOnlyList<int> UnsubscribedHandles { get { lock (_unsubscribed) return _unsubscribed.ToArray(); } }
public async Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
await Gate.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
var h = 100;
return fullReferences.Select(r => new SubscribeResult
{
TagAddress = r, ItemHandle = ++h, WasSuccessful = true,
}).ToList();
}
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
{
lock (_unsubscribed) _unsubscribed.AddRange(itemHandles);
return Task.CompletedTask;
}
public IAsyncEnumerable<ZB.MOM.WW.MxGateway.Contracts.Proto.MxEvent> StreamEventsAsync(CancellationToken cancellationToken)
=> System.Threading.Channels.Channel.CreateUnbounded<ZB.MOM.WW.MxGateway.Contracts.Proto.MxEvent>()
.Reader.ReadAllAsync(cancellationToken);
}
}