Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs
T
Joseph Doherty f89e0cc04c fix(galaxy): #859 review — read path window, atomic promote, session epoch, per-call windows
1. ReadAsync (subscribe-once) now opens the early-event window around its
   SubscribeBulk + registration; an initial value that overtook the reply was
   dropped and the read returned BadTimeout.
2. Register/Rebind write the reverse map and promote parked events in ONE
   critical section under the park gate, and ResolveOrPark resolves under that
   gate whenever anything is held (lock-free fast path only when nothing is),
   so a later live event can no longer overtake the parked initial value. A
   resolved handle that still holds a parked list queues behind it.
3. Held events are fenced by a session epoch: each new EventPump calls
   StartNewEpoch(), discarding parked AND promoted-but-unreplayed events; an
   older pump can neither park nor replay. Item-handle reuse across a reconnect
   can no longer put an old-session value on a new tag.
4. ResubscribeAllAsync opens one window per subscription, not across the whole
   replay.
5. Promoted-but-unreplayed (ready) events count toward MaxParkedEvents.
6. Begin/End of a window run under the park gate, so N->0 discard and 0->1
   open are atomic (no inherited leftovers).
Documented that ParkedEventsReady may run under _registryGate/_syncLock and
relies on the channel's AllowSynchronousContinuations=false.

Tests use TestWait (#858); each new test was checked red against a mutation
of the fix it pins.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 10:26:28 -05:00

498 lines
23 KiB
C#

using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// #859 — MXAccess's initial value for a freshly advised item can reach the event stream
/// before the SubscribeBulk reply is registered: mxaccessgw frames the reply on a thread-pool
/// continuation while the STA is already pumping the OnDataChange, and the reply and the event
/// ride separate RPCs. Such an event used to be dropped as "stale after unsubscribe", leaving
/// the node at its pre-subscribe value until the next change. These tests deliver the event
/// strictly BEFORE the reply and pin that it is parked and replayed in order, while a genuinely
/// stale event (no subscribe in flight) is still dropped.
/// </summary>
public sealed class EarlyEventWindowTests
{
private static MxEvent DataChange(int handle, double value) => new()
{
Family = MxEventFamily.OnDataChange,
ItemHandle = handle,
Value = new MxValue { DoubleValue = value },
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
};
private static GalaxyDriverOptions Opts() => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
new GalaxyMxAccessOptions("OtOpcUa-A"),
new GalaxyRepositoryOptions(),
new GalaxyReconnectOptions());
// ===== Registry =====
/// <summary>With no subscribe in flight an unknown handle is stale — the pre-#859 drop is kept.</summary>
[Fact]
public void Unknown_handle_without_a_pending_subscribe_is_stale()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
registry.ResolveOrPark(DataChange(7, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Stale);
registry.ParkedEventCount.ShouldBe(0);
}
/// <summary>Parked events are promoted, in arrival order, when the handle is registered.</summary>
[Fact]
public void Parked_events_are_promoted_in_order_on_register()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
var woken = 0;
registry.ParkedEventsReady += () => woken++;
using (registry.BeginPendingSubscribe())
{
registry.ResolveOrPark(DataChange(7, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.ResolveOrPark(DataChange(7, 2), epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.ResolveOrPark(DataChange(8, 9), epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.TakeReadyEvents(epoch).ShouldBeEmpty();
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
}
woken.ShouldBe(1);
registry.TakeReadyEvents(epoch).Select(e => e.Value.DoubleValue).ShouldBe([1.0, 2.0]);
registry.TakeReadyEvents(epoch).ShouldBeEmpty();
// Handle 8 was never registered: closing the last window discarded it.
registry.ParkedEventCount.ShouldBe(0);
}
/// <summary>A reconnect/re-snapshot Rebind promotes the new handles' early events too.</summary>
[Fact]
public void Rebind_promotes_parked_events_for_the_new_handles()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
using (registry.BeginPendingSubscribe())
{
registry.ResolveOrPark(DataChange(70, 5), epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.Rebind(1, [new TagBinding("Tank.Level", 70)]);
}
registry.TakeReadyEvents(epoch).Single().ItemHandle.ShouldBe(70);
}
/// <summary>Closing one of two overlapping windows keeps the other subscribe's parked events.</summary>
[Fact]
public void Closing_one_of_two_windows_keeps_parked_events()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
var first = registry.BeginPendingSubscribe();
var second = registry.BeginPendingSubscribe();
registry.ResolveOrPark(DataChange(7, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
first.Dispose();
first.Dispose(); // idempotent — must not close the second window
registry.PendingSubscribeCount.ShouldBe(1);
registry.ParkedEventCount.ShouldBe(1);
second.Dispose();
registry.ParkedEventCount.ShouldBe(0);
registry.ResolveOrPark(DataChange(7, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Stale);
}
/// <summary>
/// Review #6: a window opening right after the previous one closed never inherits its
/// leftovers (the N→0 discard and the 0→1 open are serialised under the park gate).
/// </summary>
[Fact]
public void A_new_window_never_inherits_the_previous_windows_leftovers()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
var first = registry.BeginPendingSubscribe();
registry.ResolveOrPark(DataChange(7, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
first.Dispose();
using var second = registry.BeginPendingSubscribe();
registry.ParkedEventCount.ShouldBe(0);
registry.Register(1, [new TagBinding("Reused", 7)]);
registry.TakeReadyEvents(epoch).ShouldBeEmpty("the closed window's event must not reach the new owner of handle 7");
}
/// <summary>Parking is bounded; past the bound the event is dropped as overflow.</summary>
[Fact]
public void Parking_is_bounded()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
using var window = registry.BeginPendingSubscribe();
var ev = DataChange(7, 1);
for (var i = 0; i < SubscriptionRegistry.MaxParkedEvents; i++)
{
registry.ResolveOrPark(ev, epoch, out _).ShouldBe(EarlyEventOutcome.Parked);
}
registry.ResolveOrPark(ev, epoch, out _).ShouldBe(EarlyEventOutcome.Overflow);
registry.ParkedEventCount.ShouldBe(SubscriptionRegistry.MaxParkedEvents);
}
/// <summary>Review #5: promoted-but-not-yet-replayed events still count toward the bound.</summary>
[Fact]
public void Ready_events_count_toward_the_bound()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
using var window = registry.BeginPendingSubscribe();
var ev = DataChange(7, 1);
for (var i = 0; i < SubscriptionRegistry.MaxParkedEvents; i++) registry.ResolveOrPark(ev, epoch, out _);
registry.Register(1, [new TagBinding("Tank.Level", 7)]); // all promoted to ready
registry.ResolveOrPark(DataChange(8, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Overflow);
registry.TakeReadyEvents(epoch).Count.ShouldBe(SubscriptionRegistry.MaxParkedEvents);
registry.ParkedEventCount.ShouldBe(0);
registry.ResolveOrPark(DataChange(8, 1), epoch, out _).ShouldBe(EarlyEventOutcome.Parked,
"positive control: once replayed, the room is back");
}
/// <summary>A registered handle resolves normally even while a window is open.</summary>
[Fact]
public void Registered_handle_resolves_while_a_window_is_open()
{
var registry = new SubscriptionRegistry();
var epoch = registry.StartNewEpoch();
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
using var window = registry.BeginPendingSubscribe();
registry.ResolveOrPark(DataChange(7, 1), epoch, out var subs).ShouldBe(EarlyEventOutcome.Resolved);
subs.Single().FullReference.ShouldBe("Tank.Level");
}
/// <summary>
/// Review #3: held events belong to a session epoch. A new epoch (a new pump after a
/// reconnect) discards parked AND promoted-but-unreplayed events, and an older pump can
/// neither park nor replay — so a reused handle number never receives an old-session value.
/// </summary>
[Fact]
public void A_new_epoch_discards_held_events_and_fences_the_old_epoch()
{
var registry = new SubscriptionRegistry();
var oldEpoch = registry.StartNewEpoch();
using var window = registry.BeginPendingSubscribe();
registry.ResolveOrPark(DataChange(7, 111), oldEpoch, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.ResolveOrPark(DataChange(8, 112), oldEpoch, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.Register(1, [new TagBinding("Old.A", 8)]); // 8 promoted to ready, never replayed
var newEpoch = registry.StartNewEpoch();
registry.ParkedEventCount.ShouldBe(0);
registry.TakeReadyEvents(newEpoch).ShouldBeEmpty();
registry.Register(2, [new TagBinding("New.B", 7)]); // new session reuses handle 7
registry.TakeReadyEvents(newEpoch).ShouldBeEmpty("the old session's value must not reach New.B");
registry.ResolveOrPark(DataChange(9, 1), oldEpoch, out _).ShouldBe(EarlyEventOutcome.Stale);
registry.TakeReadyEvents(oldEpoch).ShouldBeEmpty();
registry.ResolveOrPark(DataChange(9, 1), newEpoch, out _).ShouldBe(EarlyEventOutcome.Parked,
"positive control: the current epoch still parks");
}
// ===== EventPump =====
private sealed class ChannelSubscriber : IGalaxySubscriber
{
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
=> Task.FromResult<IReadOnlyList<SubscribeResult>>([]);
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
public void Emit(MxEvent ev) => _events.Writer.TryWrite(ev).ShouldBeTrue();
}
/// <summary>
/// An initial value delivered before registration reaches the handler once the handle is
/// registered — with NO further event on the stream (the wake path).
/// </summary>
[Fact]
public async Task Pump_replays_an_early_event_once_the_handle_registers()
{
var subscriber = new ChannelSubscriber();
var registry = new SubscriptionRegistry();
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
await using var pump = new EventPump(subscriber, registry, channelCapacity: 16, clientName: "Early");
pump.OnDataChange += (_, e) => delivered.Enqueue(e);
pump.Start();
using (registry.BeginPendingSubscribe())
{
subscriber.Emit(DataChange(7, 42));
await TestWait.UntilAsync(() => registry.ParkedEventCount == 1, "the early event must be parked, not dropped");
delivered.ShouldBeEmpty();
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
}
await TestWait.UntilAsync(() => !delivered.IsEmpty, "the parked initial value must be replayed");
var args = delivered.Single();
args.FullReference.ShouldBe("Tank.Level");
((double)args.Snapshot.Value!).ShouldBe(42.0);
}
/// <summary>The replayed early value precedes a later live value for the same handle.</summary>
[Fact]
public async Task Pump_replays_early_events_before_later_live_events()
{
var subscriber = new ChannelSubscriber();
var registry = new SubscriptionRegistry();
var delivered = new ConcurrentQueue<double>();
await using var pump = new EventPump(subscriber, registry, channelCapacity: 16, clientName: "Early");
pump.OnDataChange += (_, e) => delivered.Enqueue((double)e.Snapshot.Value!);
pump.Start();
using (registry.BeginPendingSubscribe())
{
subscriber.Emit(DataChange(7, 1));
subscriber.Emit(DataChange(7, 2));
await TestWait.UntilAsync(() => registry.ParkedEventCount == 2, "both early events must be parked");
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
subscriber.Emit(DataChange(7, 3));
}
await TestWait.UntilAsync(() => delivered.Count >= 3, "all three values must be delivered",
() => string.Join(",", delivered));
delivered.ShouldBe([1.0, 2.0, 3.0]);
}
/// <summary>
/// Review #2: a live event arriving while a registration has written the reverse map but not
/// yet promoted (a 50k-binding Register is milliseconds of work) must NOT overtake the parked
/// initial value. The hook runs inside that window, emits the newer value and gives the pump
/// ample time to (wrongly) dispatch it; the pump must instead wait for the promotion.
/// </summary>
[Fact]
public async Task Pump_does_not_let_a_live_event_overtake_a_registration_in_progress()
{
var subscriber = new ChannelSubscriber();
var registry = new SubscriptionRegistry();
var delivered = new ConcurrentQueue<double>();
await using var pump = new EventPump(subscriber, registry, channelCapacity: 16, clientName: "Early");
pump.OnDataChange += (_, e) => delivered.Enqueue((double)e.Snapshot.Value!);
pump.Start();
var deliveredDuringRegistration = -1;
using (registry.BeginPendingSubscribe())
{
subscriber.Emit(DataChange(7, 1));
await TestWait.UntilAsync(() => registry.ParkedEventCount == 1, "the initial value must be parked");
registry.AfterMapWriteForTests = () =>
{
subscriber.Emit(DataChange(7, 2));
Thread.Sleep(300);
deliveredDuringRegistration = delivered.Count;
};
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
registry.AfterMapWriteForTests = null;
}
await TestWait.UntilAsync(() => delivered.Count >= 2, "both values must be delivered",
() => string.Join(",", delivered));
deliveredDuringRegistration.ShouldBe(0, "nothing may dispatch for the handle mid-registration");
delivered.ShouldBe([1.0, 2.0]);
}
/// <summary>
/// A genuinely stale event (no subscribe in flight) is still dropped; the positive control
/// that follows it proves the pump processed it rather than not having got to it yet.
/// </summary>
[Fact]
public async Task Pump_still_drops_a_stale_event_outside_any_window()
{
var subscriber = new ChannelSubscriber();
var registry = new SubscriptionRegistry();
registry.Register(1, [new TagBinding("Tank.Level", 7)]);
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
await using var pump = new EventPump(subscriber, registry, channelCapacity: 16, clientName: "Early");
pump.OnDataChange += (_, e) => delivered.Enqueue(e);
pump.Start();
subscriber.Emit(DataChange(99, 5)); // unsubscribed handle
subscriber.Emit(DataChange(7, 6)); // positive control, strictly after
await TestWait.UntilAsync(() => !delivered.IsEmpty, "the positive control must arrive");
delivered.Single().FullReference.ShouldBe("Tank.Level");
registry.ParkedEventCount.ShouldBe(0);
// Registering 99 now must not resurrect the stale event.
registry.Register(2, [new TagBinding("Other", 99)]);
registry.ParkedEventCount.ShouldBe(0);
}
/// <summary>
/// Review #3, pump level: an event the old session's pump parked is never replayed by the
/// next session's pump onto the tag that now owns the reused handle number.
/// </summary>
[Fact]
public async Task Pump_after_reconnect_never_replays_an_old_session_event_onto_a_reused_handle()
{
var registry = new SubscriptionRegistry();
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
using var window = registry.BeginPendingSubscribe(); // e.g. a subscribe hanging across the reconnect
var oldStream = new ChannelSubscriber();
var oldPump = new EventPump(oldStream, registry, channelCapacity: 16, clientName: "Old");
oldPump.Start();
oldStream.Emit(DataChange(7, 111));
await TestWait.UntilAsync(() => registry.ParkedEventCount == 1, "the old session's event must be parked");
await oldPump.DisposeAsync();
var newStream = new ChannelSubscriber();
await using var newPump = new EventPump(newStream, registry, channelCapacity: 16, clientName: "New");
newPump.OnDataChange += (_, e) => delivered.Enqueue(e);
newPump.Start();
registry.Register(2, [new TagBinding("New.Tag", 7)]); // new session reuses handle 7
newStream.Emit(DataChange(7, 222)); // positive control on the new session
await TestWait.UntilAsync(() => !delivered.IsEmpty, "the new session's value must arrive");
await TestWait.UntilAsync(() => registry.ParkedEventCount == 0, "nothing may stay held");
delivered.Select(e => (double)e.Snapshot.Value!).ShouldBe([222.0]);
}
// ===== GalaxyDriver / probe watcher end-to-end =====
/// <summary>
/// A gateway fake that writes each new handle's initial value to the event stream and only
/// returns the SubscribeBulk reply once the consumer's pump has parked that event — the
/// event deterministically precedes the reply.
/// </summary>
private sealed class EventBeforeReplySubscriber : IGalaxySubscriber
{
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
private int _nextHandle = 100;
/// <summary>Gets or sets the consumer's held-event count, polled before replying.</summary>
public Func<int> ParkedCount { get; set; } = () => 0;
/// <summary>Gets the handles unsubscribed so far.</summary>
public ConcurrentQueue<int> Unsubscribed { get; } = new();
public async Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
{
var results = new List<SubscribeResult>();
var before = ParkedCount();
foreach (var fullRef in fullReferences)
{
var handle = Interlocked.Increment(ref _nextHandle);
await _events.Writer.WriteAsync(DataChange(handle, handle * 10.0), cancellationToken);
results.Add(new SubscribeResult { TagAddress = fullRef, ItemHandle = handle, WasSuccessful = true });
}
await TestWait.UntilAsync(() => ParkedCount() >= before + fullReferences.Count,
"the initial values must reach the pump (and be parked) before the reply is returned",
() => $"held={ParkedCount()} before={before}");
return results;
}
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
{
foreach (var h in itemHandles) Unsubscribed.Enqueue(h);
return Task.CompletedTask;
}
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
}
/// <summary>The driver publishes MXAccess's initial value even when it overtook the subscribe reply.</summary>
[Fact]
public async Task Driver_publishes_an_initial_value_that_arrived_before_the_subscribe_reply()
{
var subscriber = new EventBeforeReplySubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
subscriber.ParkedCount = () => driver.ParkedEarlyEventCountForTests;
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
driver.OnDataChange += (_, e) => delivered.Enqueue(e);
var handle = await driver.SubscribeAsync(
["Tank.Level", "Tank.Temp"], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken);
await TestWait.UntilAsync(() => delivered.Count >= 2, "both initial values must be published");
delivered.Select(e => e.FullReference).OrderBy(r => r).ShouldBe(["Tank.Level", "Tank.Temp"]);
delivered.ShouldAllBe(e => e.SubscriptionHandle.Equals(handle));
driver.ParkedEarlyEventCountForTests.ShouldBe(0);
}
/// <summary>
/// Review #1: the subscribe-once read waits ONLY for the initial value, so an initial value
/// that overtakes the reply must still complete the read Good — not BadTimeout.
/// </summary>
[Fact]
public async Task Driver_read_returns_an_initial_value_that_arrived_before_the_subscribe_reply()
{
var subscriber = new EventBeforeReplySubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
subscriber.ParkedCount = () => driver.ParkedEarlyEventCountForTests;
var snapshots = await TestWait.ForAsync(
driver.ReadAsync(["Tank.Level"], TestContext.Current.CancellationToken),
"the read must complete");
var snapshot = snapshots.ShouldHaveSingleItem();
snapshot.StatusCode.ShouldBe(StatusCodeMap.Good);
((double)snapshot.Value!).ShouldBe(1010.0);
driver.ParkedEarlyEventCountForTests.ShouldBe(0);
}
/// <summary>The probe watcher's initial ScanState overtaking its reply still reaches the fan-out.</summary>
[Fact]
public async Task Probe_watcher_initial_value_that_arrived_before_the_reply_is_dispatched()
{
var registry = new SubscriptionRegistry();
var subscriber = new EventBeforeReplySubscriber { ParkedCount = () => registry.ParkedEventCount };
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
await using var pump = new EventPump(subscriber, registry, channelCapacity: 16, clientName: "Probe");
pump.OnDataChange += (_, e) => delivered.Enqueue(e);
pump.Start();
using var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
await watcher.SyncPlatformsAsync(["PlatformA"], TestContext.Current.CancellationToken);
await TestWait.UntilAsync(() => !delivered.IsEmpty, "the initial ScanState must be dispatched");
var args = delivered.Single();
args.FullReference.ShouldBe("PlatformA.ScanState");
((GalaxySubscriptionHandle)args.SubscriptionHandle).SubscriptionId.ShouldBe(SubscriptionRegistry.ProbeSubscriptionId);
}
}