4fdc3708f4
Under heavy parallel load (14 concurrent test processes + 14 CPU burners, with DOTNET_ThreadPool_ForceMinWorkerThreads=1 as the reproducer) the Galaxy tests with 1-3 s fixed waits failed 16/70 runs. Every failure was a background loop started with Task.Run (alarm feed, DeployWatcher, EventPump, ReconnectSupervisor) not being SCHEDULED within the ceiling - e.g. DeployWatcher's stream had not even been opened after 2 s. No production race: the code under test was checked for each (alarm feed dispatch is synchronous in-order, DeployWatcher/ReconnectSupervisor state is lock- or loop-local, subscribe registers before returning). - New shared TestWait (UntilAsync / ForAsync / HealthyAsync): 30 s presence ceiling, failure reports elapsed ms + an observed-state snapshot (#856 shape; GalaxyDriverFixRound1Tests.WaitUntilAsync now delegates to it). - Converted the short waits in GatewayGalaxyAlarmFeedTests, DeployWatcherTests, GalaxyDriverSubscribeTests, GalaxyDriverRawPathKeyingTests (its WaitForAsync never threw on timeout), EventPumpStreamFaultTests, EventPumpReplayGapTests, ReconnectSupervisorTests; FixRound2 InitBudget 2 s -> 10 s (the parked fetch never completes, so any finite bound keeps the verdict). - Absence by short sleep replaced with positive controls: DeployWatcher bootstrap and same-deploy-time (follow with a real change, assert exactly that one raise - verified red when suppression is removed), subscribe failed-tag / unsubscribed handle (follow with a live handle's event), faulted pump (wait for its onStreamFault instead of 100 ms). - Fixed sleeps that raced scheduling: subscribe-once read tests wait for the read's registration (TrackedSubscriptionCountForTests) instead of 50 ms; supervisor Dispose test waits for reopen entry instead of 50 ms; the already-Healthy WaitForHealthy test asserts synchronous completion instead of a wall-clock bound. FakeSubscriber.Map is now a ConcurrentDictionary. Loaded run (14 parallel x 5, 14 burners, ForceMinWorkerThreads=1): before 16/70 failed runs; after 0/350 (70 + 210 + 70). Without the thread-pool override: 0/70 before and after on this host. Serial: 472 passed, 8 skipped. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
342 lines
17 KiB
C#
342 lines
17 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Threading.Channels;
|
|
using Google.Protobuf.WellKnownTypes;
|
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
|
|
|
/// <summary>
|
|
/// End-to-end tests for <see cref="GalaxyDriver"/>'s ISubscribable wiring +
|
|
/// <see cref="EventPump"/>. The fake subscriber replays a controlled stream of
|
|
/// <see cref="MxEvent"/>s; the test asserts the driver's <c>OnDataChange</c> fans
|
|
/// out per registered subscription.
|
|
/// </summary>
|
|
public sealed class GalaxyDriverSubscribeTests
|
|
{
|
|
private static GalaxyDriverOptions Opts() => new(
|
|
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
|
new GalaxyMxAccessOptions("OtOpcUa-A"),
|
|
new GalaxyRepositoryOptions(),
|
|
new GalaxyReconnectOptions());
|
|
|
|
internal sealed class FakeSubscriber : IGalaxySubscriber
|
|
{
|
|
private int _nextHandle = 1;
|
|
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
|
|
|
/// <summary>Gets the mapping of tag references to subscription handles.</summary>
|
|
/// <remarks>Concurrent: the driver may run SubscribeBulk on another thread while a test polls it.</remarks>
|
|
public ConcurrentDictionary<string, int> Map { get; } = new();
|
|
|
|
/// <summary>Gets the list of unsubscribed handles.</summary>
|
|
public List<int> UnsubscribedHandles { get; } = [];
|
|
|
|
/// <summary>Gets the list of buffered intervals called.</summary>
|
|
public List<int> BufferedIntervalsCalled { get; } = [];
|
|
|
|
/// <summary>Gets or sets a function to decide whether to accept a subscription.</summary>
|
|
public Func<string, bool> Decide { get; set; } = _ => true;
|
|
|
|
/// <summary>Subscribes to bulk updates for the specified tag references.</summary>
|
|
/// <param name="fullReferences">The tag references to subscribe to.</param>
|
|
/// <param name="bufferedUpdateIntervalMs">The buffered update interval in milliseconds.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
/// <returns>A list of subscription results.</returns>
|
|
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
|
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
|
{
|
|
BufferedIntervalsCalled.Add(bufferedUpdateIntervalMs);
|
|
var results = new List<SubscribeResult>(fullReferences.Count);
|
|
foreach (var fullRef in fullReferences)
|
|
{
|
|
if (Decide(fullRef))
|
|
{
|
|
var handle = Interlocked.Increment(ref _nextHandle);
|
|
Map[fullRef] = handle;
|
|
results.Add(new SubscribeResult
|
|
{
|
|
TagAddress = fullRef,
|
|
ItemHandle = handle,
|
|
WasSuccessful = true,
|
|
});
|
|
}
|
|
else
|
|
{
|
|
results.Add(new SubscribeResult
|
|
{
|
|
TagAddress = fullRef,
|
|
ItemHandle = 0,
|
|
WasSuccessful = false,
|
|
ErrorMessage = "rejected by fake",
|
|
});
|
|
}
|
|
}
|
|
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
|
}
|
|
|
|
/// <summary>Unsubscribes from bulk updates for the specified item handles.</summary>
|
|
/// <param name="itemHandles">The handles to unsubscribe.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
/// <returns>A completed task.</returns>
|
|
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
|
{
|
|
UnsubscribedHandles.AddRange(itemHandles);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Streams events asynchronously.</summary>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
/// <returns>An async enumerable of MX events.</returns>
|
|
public IAsyncEnumerable<MxEvent> StreamEventsAsync(CancellationToken cancellationToken)
|
|
=> _events.Reader.ReadAllAsync(cancellationToken);
|
|
|
|
/// <summary>Emits a data change event asynchronously.</summary>
|
|
/// <param name="itemHandle">The handle of the item that changed.</param>
|
|
/// <param name="value">The new value.</param>
|
|
/// <param name="quality">The quality of the value.</param>
|
|
/// <returns>A value task representing the asynchronous emission.</returns>
|
|
public ValueTask EmitOnDataChangeAsync(int itemHandle, double value, byte quality = 192) =>
|
|
_events.Writer.WriteAsync(new MxEvent
|
|
{
|
|
Family = MxEventFamily.OnDataChange,
|
|
ItemHandle = itemHandle,
|
|
Value = new MxValue { DoubleValue = value },
|
|
Quality = quality,
|
|
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
|
});
|
|
|
|
/// <summary>Completes the event stream.</summary>
|
|
public void CompleteEvents() => _events.Writer.Complete();
|
|
}
|
|
|
|
/// <summary>Verifies subscription allocates a handle and dispatches value changes.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_AllocatesHandle_AndDispatchesValueChange()
|
|
{
|
|
var subscriber = new FakeSubscriber();
|
|
using var driver = new GalaxyDriver(
|
|
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
var captured = new List<DataChangeEventArgs>();
|
|
driver.OnDataChange += (_, args) => { lock (captured) captured.Add(args); };
|
|
|
|
var handle = await driver.SubscribeAsync(["Tank.Level"], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
|
|
var itemHandle = subscriber.Map["Tank.Level"];
|
|
await subscriber.EmitOnDataChangeAsync(itemHandle, 42.0);
|
|
|
|
await WaitForCapturedAsync(captured, 1, "the value change must be dispatched");
|
|
captured.Count.ShouldBe(1);
|
|
captured[0].SubscriptionHandle.ShouldBe(handle);
|
|
captured[0].FullReference.ShouldBe("Tank.Level");
|
|
((double)captured[0].Snapshot.Value!).ShouldBe(42.0);
|
|
}
|
|
|
|
/// <summary>Verifies two subscriptions for the same tag each receive updates.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_TwoSubscriptions_SameTag_FanOutOnePerSubscription()
|
|
{
|
|
var subscriber = new FakeSubscriber();
|
|
using var driver = new GalaxyDriver(
|
|
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
var captured = new List<DataChangeEventArgs>();
|
|
driver.OnDataChange += (_, args) => { lock (captured) captured.Add(args); };
|
|
|
|
var handle1 = await driver.SubscribeAsync(["A"], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
var handle2 = await driver.SubscribeAsync(["A"], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
|
|
// Both subscriptions resolved the same FullRef. The fake gives each its own
|
|
// itemHandle (Map["A"] gets overwritten), so we use the latest mapping for the
|
|
// second subscription's expected delivery; the first subscription's binding
|
|
// points at an item handle the gateway fake hasn't emitted on. To exercise the
|
|
// fan-out, register both subs against the SAME handle (matches the gw's "one
|
|
// handle per (server, tag) pair" pattern in production where SubscribeBulk
|
|
// returns the existing handle for an already-AddItem'd tag).
|
|
subscriber.Map["A"].ShouldBeGreaterThan(0);
|
|
// Synthesize an event against handle 2 (which is also tracked under sub 2).
|
|
// Fan-out for the same tag is best validated at the registry level — the
|
|
// SubscriptionRegistryTests cover the multi-sub-same-handle case directly.
|
|
await subscriber.EmitOnDataChangeAsync(subscriber.Map["A"], 7.0);
|
|
|
|
await WaitForCapturedAsync(captured, 1, "the value change must reach a subscription");
|
|
|
|
// At least one delivery — depending on which subscription owns the handle,
|
|
// either handle1 or handle2 receives. The fan-out invariant (a single handle
|
|
// delivers to every subscription that registered it) is pinned in
|
|
// SubscriptionRegistryTests; here we just confirm the wiring works.
|
|
captured.ShouldNotBeEmpty();
|
|
captured[0].SubscriptionHandle.DiagnosticId.ShouldStartWith("galaxy-sub-");
|
|
// Either handle1 or handle2 must match the captured handle's id.
|
|
var captured0Id = ((GalaxySubscriptionHandle)captured[0].SubscriptionHandle).SubscriptionId;
|
|
var allowed = new[] {
|
|
((GalaxySubscriptionHandle)handle1).SubscriptionId,
|
|
((GalaxySubscriptionHandle)handle2).SubscriptionId,
|
|
};
|
|
allowed.ShouldContain(captured0Id);
|
|
}
|
|
|
|
/// <summary>Verifies failed subscriptions do not dispatch events.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_FailedTag_DoesNotDispatchEvents()
|
|
{
|
|
var subscriber = new FakeSubscriber { Decide = tag => tag != "Bad" };
|
|
using var driver = new GalaxyDriver(
|
|
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
var captured = new List<DataChangeEventArgs>();
|
|
driver.OnDataChange += (_, args) => { lock (captured) captured.Add(args); };
|
|
|
|
await driver.SubscribeAsync(["Good", "Bad"], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
|
|
// Good has an itemHandle; Bad doesn't (item handle 0). An event with handle 0
|
|
// must NOT be dispatched (no subscribers registered against it).
|
|
await subscriber.EmitOnDataChangeAsync(itemHandle: 0, value: 999.0);
|
|
// Positive control (#858): the pump dispatches in stream order, so once Good's event is seen the
|
|
// handle-0 event has been handled — its absence is a real result, not a short sleep.
|
|
await subscriber.EmitOnDataChangeAsync(subscriber.Map["Good"], 1.0);
|
|
await WaitForCapturedAsync(captured, 1, "positive control: Good's event must be dispatched");
|
|
|
|
var only = Snapshot(captured).ShouldHaveSingleItem("the handle-0 event must not be dispatched");
|
|
only.FullReference.ShouldBe("Good");
|
|
}
|
|
|
|
/// <summary>Verifies unsubscribe removes registration and calls gateway unsubscribe.</summary>
|
|
[Fact]
|
|
public async Task UnsubscribeAsync_RemovesRegistration_AndCallsGwUnsubscribe()
|
|
{
|
|
var subscriber = new FakeSubscriber();
|
|
using var driver = new GalaxyDriver(
|
|
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
var handle = await driver.SubscribeAsync(["X"], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
var itemHandle = subscriber.Map["X"];
|
|
// A second, still-live subscription provides the positive control below.
|
|
await driver.SubscribeAsync(["Control"], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
|
|
await driver.UnsubscribeAsync(handle, CancellationToken.None);
|
|
|
|
subscriber.UnsubscribedHandles.ShouldContain(itemHandle);
|
|
|
|
// Subsequent events for the dropped handle don't dispatch.
|
|
var captured = new List<DataChangeEventArgs>();
|
|
driver.OnDataChange += (_, args) => { lock (captured) captured.Add(args); };
|
|
await subscriber.EmitOnDataChangeAsync(itemHandle, 11.0);
|
|
// Positive control (#858): events dispatch in stream order, so once the live subscription's event
|
|
// is seen the dropped handle's event has been handled.
|
|
await subscriber.EmitOnDataChangeAsync(subscriber.Map["Control"], 12.0);
|
|
await WaitForCapturedAsync(captured, 1, "positive control: the live subscription's event must be dispatched");
|
|
|
|
var only = Snapshot(captured).ShouldHaveSingleItem("the unsubscribed handle's event must not be dispatched");
|
|
only.FullReference.ShouldBe("Control");
|
|
}
|
|
|
|
/// <summary>Verifies unsubscribing with an unknown handle is handled.</summary>
|
|
[Fact]
|
|
public async Task UnsubscribeAsync_UnknownHandle_NoOp()
|
|
{
|
|
var subscriber = new FakeSubscriber();
|
|
using var driver = new GalaxyDriver(
|
|
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
// Handle issued by a different driver shape — must throw (it's a programming
|
|
// error, not a recoverable runtime condition).
|
|
var foreignHandle = new ForeignHandle();
|
|
await Should.ThrowAsync<ArgumentException>(() =>
|
|
driver.UnsubscribeAsync(foreignHandle, CancellationToken.None));
|
|
}
|
|
|
|
/// <summary>Verifies subscription without a subscriber throws.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_NoSubscriber_Throws()
|
|
{
|
|
using var driver = new GalaxyDriver("g", Opts(), hierarchySource: null);
|
|
var ex = await Should.ThrowAsync<NotSupportedException>(() =>
|
|
driver.SubscribeAsync(["x"], TimeSpan.FromSeconds(1), CancellationToken.None));
|
|
ex.Message.ShouldContain("GalaxyDriver.SubscribeAsync requires");
|
|
}
|
|
|
|
/// <summary>Verifies subscription falls back to configured interval when zero is passed.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_FallsBackToConfiguredInterval_WhenCallerPassesZero()
|
|
{
|
|
// PR 6.3 — when the caller doesn't set a publishing interval (TimeSpan.Zero),
|
|
// the driver substitutes MxAccess.PublishingIntervalMs from its options.
|
|
var subscriber = new FakeSubscriber();
|
|
var opts = new GalaxyDriverOptions(
|
|
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
|
new GalaxyMxAccessOptions("OtOpcUa-A", PublishingIntervalMs: 750),
|
|
new GalaxyRepositoryOptions(),
|
|
new GalaxyReconnectOptions());
|
|
using var driver = new GalaxyDriver(
|
|
"g", opts, hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
await driver.SubscribeAsync(["Tag.A"], TimeSpan.Zero, CancellationToken.None);
|
|
|
|
subscriber.BufferedIntervalsCalled.ShouldHaveSingleItem().ShouldBe(750);
|
|
}
|
|
|
|
/// <summary>Verifies subscription respects caller's interval when non-zero.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_RespectsCallerInterval_WhenNonZero()
|
|
{
|
|
// The caller's publishingInterval wins when explicitly set — the configured
|
|
// option only applies as a fallback for "no-preference" callers.
|
|
var subscriber = new FakeSubscriber();
|
|
var opts = new GalaxyDriverOptions(
|
|
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
|
new GalaxyMxAccessOptions("OtOpcUa-A", PublishingIntervalMs: 750),
|
|
new GalaxyRepositoryOptions(),
|
|
new GalaxyReconnectOptions());
|
|
using var driver = new GalaxyDriver(
|
|
"g", opts, hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
await driver.SubscribeAsync(["Tag.A"], TimeSpan.FromMilliseconds(250), CancellationToken.None);
|
|
|
|
subscriber.BufferedIntervalsCalled.ShouldHaveSingleItem().ShouldBe(250);
|
|
}
|
|
|
|
/// <summary>Verifies subscription with empty tag list returns handle without calling gateway.</summary>
|
|
[Fact]
|
|
public async Task SubscribeAsync_EmptyTagList_ReturnsHandleWithoutCallingGw()
|
|
{
|
|
var subscriber = new FakeSubscriber();
|
|
using var driver = new GalaxyDriver(
|
|
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
|
|
|
var handle = await driver.SubscribeAsync([], TimeSpan.FromSeconds(1), CancellationToken.None);
|
|
handle.ShouldNotBeNull();
|
|
subscriber.Map.ShouldBeEmpty();
|
|
}
|
|
|
|
/// <summary>A subscription handle from a foreign source.</summary>
|
|
private sealed class ForeignHandle : ISubscriptionHandle
|
|
{
|
|
/// <summary>Gets the diagnostic identifier for this handle.</summary>
|
|
public string DiagnosticId => "foreign-x";
|
|
}
|
|
|
|
/// <summary>Presence wait (#858): until <paramref name="captured"/> holds at least <paramref name="count"/> events.</summary>
|
|
internal static Task WaitForCapturedAsync(List<DataChangeEventArgs> captured, int count, string because) =>
|
|
TestWait.UntilAsync(
|
|
() => { lock (captured) return captured.Count >= count; },
|
|
because,
|
|
() =>
|
|
{
|
|
lock (captured)
|
|
{
|
|
return $"dispatched={captured.Count}/{count} [{string.Join(", ", captured.Select(c => $"{c.FullReference}={c.Snapshot.Value}"))}]";
|
|
}
|
|
});
|
|
|
|
internal static List<DataChangeEventArgs> Snapshot(List<DataChangeEventArgs> captured)
|
|
{
|
|
lock (captured) return [.. captured];
|
|
}
|
|
}
|