using System.Diagnostics.Metrics; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// PR 6.2 — pins the EventPump's bounded-channel + drop-newest behavior. We hold the dispatch loop /// with a blocking handler so the channel fills, then verify the producer keeps reading from the gw /// stream and increments the galaxy.events.dropped counter rather than blocking. /// /// /// These tests observe two background loops (RunAsync / DispatchLoopAsync) through a /// process-wide , so every wait here is a presence assertion polled up to /// — never a fixed sleep. A fixed sleep is a bet that both loops get scheduled /// within it, and #503 is what losing that bet looks like when the box is busy running other test /// assemblies. /// public sealed class EventPumpBoundedChannelTests { /// Upper bound on how long a poll waits for a background loop to reach its terminal state. /// Generous on purpose: it is spent only when something is genuinely broken, and it cannot turn a /// failing assertion into a passing one. private static readonly TimeSpan Budget = TimeSpan.FromSeconds(15); /// /// With the dispatch loop genuinely held and the channel therefore full, the producer keeps /// draining the gw stream and counts every rejected write on galaxy.events.dropped — /// newest-dropped, not back-pressure. /// /// /// /// The arrangement is staged so the arithmetic is exact rather than scheduler-dependent /// (#503): emit ONE event, wait until the dispatcher has actually entered the handler (so it /// holds that event and the channel is empty), and only then emit the remaining nine. The /// channel accepts capacity of them and the rest must be dropped — /// 10 − 1 held − 2 buffered = 7, every run. /// /// [Fact] public async Task Drops_newest_when_channel_fills_and_records_metric() { const int totalEvents = 10; const int channelCapacity = 2; // One event is held inside the handler; `channelCapacity` more fit in the channel; the rest are dropped. const int expectedDropped = totalEvents - 1 - channelCapacity; // Unique per run: the counters are filtered to this exact galaxy.client tag, which is what keeps a // parallel sibling test's pump out of them (#503). A literal name would work today and break the // day someone reuses it. var clientName = $"PumpTest-{Guid.NewGuid():N}"; var counters = StartMeterCapture(clientName); // A SYNCHRONOUS gate, deliberately. OnDataChange is EventHandler — void-returning — so an // `async (_, _) => await gate` lambda is async void: it returns to the dispatch loop at the first // await and holds nothing. That is exactly what #503 was: the loop drained freely, drops happened // only when the producer happened to outrun the consumer, and the assertions rode on scheduling // luck. Blocking the loop's own thread is what makes "the dispatcher is held" true. var dispatchGate = new ManualResetEventSlim(false); var handlerEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); EventPump? pump = null; try { var subscriber = new ManualSubscriber(); var registry = new SubscriptionRegistry(); registry.Register(1, [new TagBinding("Tag.A", ItemHandle: 7)]); pump = new EventPump( subscriber, registry, channelCapacity: channelCapacity, clientName: clientName); pump.OnDataChange += (_, _) => { handlerEntered.TrySetResult(); dispatchGate.Wait(); }; pump.Start(); // Stage 1 — one event, then wait for the dispatcher to take it and block. After this the // channel is provably empty and exactly one event is in flight inside the handler. await subscriber.EmitAsync(itemHandle: 7, value: 0); await handlerEntered.Task.WaitAsync(Budget); // Stage 2 — the rest. `channelCapacity` are accepted; the remainder must be dropped. for (var i = 1; i < totalEvents; i++) { await subscriber.EmitAsync(itemHandle: 7, value: i); } // PRESENCE assertion on the discriminator (#500's rule): poll until the drop count reaches its // terminal value rather than sleeping a fixed 150 ms and hoping the producer got scheduled. // The budget is an upper bound before giving up — spending it costs nothing on the happy path, // and no amount of it can make a genuinely broken pump pass. Dropped starts at 0, so this // cannot be satisfied by the initial state. await WaitUntilAsync(() => counters.Dropped == expectedDropped, $"expected {expectedDropped} dropped events"); counters.Received.ShouldBe(totalEvents, "the producer must keep reading the gw stream through the overflow, not back-pressure"); counters.Dropped.ShouldBe(expectedDropped); counters.Dispatched.ShouldBe(0, "the dispatch loop is still blocked in the handler, so nothing has completed dispatch — " + "if this is non-zero the handler is not actually holding the loop and the test proves nothing"); } finally { // Release BEFORE disposing: DisposeAsync awaits the dispatch loop, and that loop is parked on // this gate. Releasing it here (not at the end of the try) is what keeps a failed assertion a // failure instead of a hang. dispatchGate.Set(); if (pump is not null) await pump.DisposeAsync(); dispatchGate.Dispose(); counters.Dispose(); } } /// Verifies that the event pump throws an exception when the channel capacity is invalid. [Fact] public async Task Throws_when_channelCapacity_is_invalid() { var subscriber = new ManualSubscriber(); var registry = new SubscriptionRegistry(); Should.Throw(() => new EventPump(subscriber, registry, channelCapacity: 0)); Should.Throw(() => new EventPump(subscriber, registry, channelCapacity: -1)); await Task.CompletedTask; } /// Verifies that event pump metrics are tagged with the client name for tracking multiple driver hosts. [Fact] public async Task Tags_metrics_with_client_name_for_multi_driver_hosts() { var captured = new List<(string Instrument, KeyValuePair[] Tags)>(); using var listener = new MeterListener(); listener.InstrumentPublished = (instr, l) => { if (instr.Meter.Name == EventPump.MeterName) l.EnableMeasurementEvents(instr); }; // The callback fires on the thread that calls Counter.Add() — that is the // RunAsync background Task. Use lock(captured) everywhere to avoid torn reads. listener.SetMeasurementEventCallback((instr, _, tags, _) => { lock (captured) { captured.Add((instr.Name, tags.ToArray())); } }); listener.Start(); var subscriber = new ManualSubscriber(); var registry = new SubscriptionRegistry(); registry.Register(1, [new TagBinding("Tag.A", ItemHandle: 7)]); await using (var pump = new EventPump(subscriber, registry, channelCapacity: 4, clientName: "Driver-X")) { pump.Start(); await subscriber.EmitAsync(7, 42.0); // Poll until at least one galaxy.events.received measurement tagged // galaxy.client=Driver-X lands in the listener, rather than using a // fixed delay that races under parallel test load on a busy box. // Budget, not 5 s: the same contention that produced #503 stretches how long the // producer loop takes to be scheduled, and a presence poll costs nothing to over-budget. await WaitUntilAsync( () => { listener.RecordObservableInstruments(); lock (captured) { return captured.Any(c => c.Instrument == "galaxy.events.received" && c.Tags.Any(t => t.Key == "galaxy.client" && string.Equals((string?)t.Value, "Driver-X", StringComparison.Ordinal))); } }, "a galaxy.events.received measurement tagged galaxy.client=Driver-X"); } // The static Meter is shared across all EventPump instances in the test // assembly; xUnit may run other pump tests in parallel and their // measurements land on the same listener. Filter to our pump's tag value. List<(string Instrument, KeyValuePair[] Tags)> ours; lock (captured) { ours = captured .Where(c => c.Tags.Any(t => t.Key == "galaxy.client" && string.Equals((string?)t.Value, "Driver-X", StringComparison.Ordinal))) .ToList(); } ours.ShouldNotBeEmpty( "at least one measurement from this test's pump must carry galaxy.client=Driver-X"); ours.ShouldContain(c => c.Instrument == "galaxy.events.received"); } /// Polls until it holds or elapses, failing /// with on timeout. The presence-assertion counterpart to a fixed delay. private static async Task WaitUntilAsync(Func condition, string what) { var deadline = DateTime.UtcNow + Budget; while (DateTime.UtcNow < deadline) { if (condition()) return; await Task.Delay(10); } condition().ShouldBeTrue($"timed out after {Budget.TotalSeconds:0}s waiting: {what}"); } /// /// Captures the three EventPump counters for one specific pump, identified by its /// galaxy.client tag. /// /// /// /// The galaxy.client filter is the fix for #503, and it is not optional. The pump's /// is static — one instance for the whole process — and /// InstrumentPublished can only select by meter, so an unfiltered listener /// receives measurements from every alive anywhere in the assembly. /// xUnit runs test classes in parallel and several of them build pumps /// (EventPumpStreamFaultTests, the Driver-X test below), so a sibling's events /// landed in these counters — that is why the flake needed a busy box: it needed the two /// tests to overlap in time. The leak was measured directly, as Received = 11 in a run /// that emitted 10. /// /// /// The old Received == Dispatched + Dropped + InFlight assertion is what turned the /// leak into a failure: a foreign increment landing between its four separate /// calls made the two sides disagree. Hence the /// reported subject, counters.Received. /// /// /// The galaxy.client tag value to accept. Must be unique to the /// calling test — pass a fresh GUID-suffixed name rather than a literal, so a future sibling test /// cannot silently reintroduce the leak by reusing the same name. private static CounterCapture StartMeterCapture(string clientName) { var capture = new CounterCapture(); var listener = new MeterListener(); listener.InstrumentPublished = (instr, l) => { if (instr.Meter.Name == EventPump.MeterName) l.EnableMeasurementEvents(instr); }; listener.SetMeasurementEventCallback((instr, value, tags, _) => { var mine = false; foreach (var tag in tags) { if (tag.Key == "galaxy.client" && string.Equals((string?)tag.Value, clientName, StringComparison.Ordinal)) { mine = true; break; } } if (!mine) return; switch (instr.Name) { case "galaxy.events.received": Interlocked.Add(ref capture._received, value); break; case "galaxy.events.dispatched": Interlocked.Add(ref capture._dispatched, value); break; case "galaxy.events.dropped": Interlocked.Add(ref capture._dropped, value); break; } }); listener.Start(); capture.Listener = listener; return capture; } private sealed class CounterCapture : IDisposable { public MeterListener? Listener; internal long _received, _dispatched, _dropped; /// Gets the count of received events. public long Received => Interlocked.Read(ref _received); /// Gets the count of dispatched events. public long Dispatched => Interlocked.Read(ref _dispatched); /// Gets the count of dropped events. public long Dropped => Interlocked.Read(ref _dropped); // There is deliberately no InFlight member. It could only be DERIVED as // Received - Dispatched - Dropped, which made the old // `Received == Dispatched + Dropped + InFlight` assertion a tautology: it could not fail on a // real defect, only on a torn read across the four separate Interlocked.Read calls. Assert the // three measured counters directly instead (#503). /// Disposes the meter listener. public void Dispose() => Listener?.Dispose(); } private sealed class ManualSubscriber : IGalaxySubscriber { private readonly Channel _stream = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true }); /// Subscribes to a bulk list of tag references. /// The list of full references to subscribe to. /// The buffered update interval in milliseconds. /// The cancellation token. /// An empty result list. public Task> SubscribeBulkAsync( IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) => Task.FromResult>([]); /// Unsubscribes from a bulk list of item handles. /// The list of item handles to unsubscribe from. /// The cancellation token. /// A completed task. public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; /// Streams events asynchronously. /// The cancellation token. /// An async enumerable of MxEvent objects. public IAsyncEnumerable StreamEventsAsync(CancellationToken cancellationToken) => _stream.Reader.ReadAllAsync(cancellationToken); /// Emits a test event with the specified item handle and value. /// The item handle. /// The event value. /// A completed value task. public ValueTask EmitAsync(int itemHandle, double value) => _stream.Writer.WriteAsync(new MxEvent { Family = MxEventFamily.OnDataChange, ItemHandle = itemHandle, Value = new MxValue { DoubleValue = value }, Quality = 192, SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), }); } }