Retain fake worker fault diagnostics in test fixtures
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m20s
ci / java (push) Successful in 2m19s
ci / windows-x86 (pull_request) Successful in 1m20s
ci / nightly-windev (pull_request) Has been skipped
ci / java (pull_request) Successful in 2m23s
ci / portable (push) Successful in 9m10s
ci / portable (pull_request) Successful in 9m2s

This commit is contained in:
Joseph Doherty
2026-10-13 10:35:28 -04:00
parent d779daf6af
commit 83aa76c33d
5 changed files with 340 additions and 26 deletions
@@ -2,6 +2,7 @@ using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Xunit.Abstractions;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
@@ -21,7 +22,7 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
/// the fake worker harness. Covers fan-out to two concurrent subscribers, independent
/// cancellation isolation, and the per-session subscriber cap.
/// </summary>
public sealed class GatewayEndToEndMultiSubscriberTests
public sealed class GatewayEndToEndMultiSubscriberTests(ITestOutputHelper output)
{
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10);
private const int ServerHandle = 3001;
@@ -37,7 +38,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
public async Task StreamEvents_TwoSubscribers_BothReceiveAllEvents()
{
MultiEventFakeWorkerProcessLauncher launcher = new(EventCount);
await using MultiSubscriberGatewayServiceFixture fixture = new(launcher);
await using MultiSubscriberGatewayServiceFixture fixture = new(launcher, diagnosticOutput: output.WriteLine);
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
@@ -136,7 +137,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
public async Task StreamEvents_OneSubscriberCancels_OtherContinuesReceivingEvents()
{
GatedEventFakeWorkerProcessLauncher launcher = new();
await using MultiSubscriberGatewayServiceFixture fixture = new(launcher);
await using MultiSubscriberGatewayServiceFixture fixture = new(launcher, diagnosticOutput: output.WriteLine);
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
@@ -254,7 +255,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
// (EventSubscriberLimitReached) is only reachable when multi-subscriber mode is on.
// Without it the service hits EventSubscriberAlreadyActive instead — also
// ResourceExhausted, but for the wrong reason — masking a misconfiguration.
await using MultiSubscriberGatewayServiceFixture fixture = new(launcher, maxEventSubscribersPerSession: cap);
await using MultiSubscriberGatewayServiceFixture fixture = new(launcher, maxEventSubscribersPerSession: cap, diagnosticOutput: output.WriteLine);
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
@@ -413,20 +414,25 @@ public sealed class GatewayEndToEndMultiSubscriberTests
{
private readonly GatewayMetrics _metrics = new();
private readonly SessionRegistry _registry = new();
private readonly FakeWorkerDiagnostics _diagnostics;
/// <summary>Initializes a new instance of the <see cref="MultiSubscriberGatewayServiceFixture"/> class.</summary>
/// <param name="launcher">Fake worker process launcher backing the session manager.</param>
/// <param name="diagnosticOutput">Safe metadata output sink.</param>
/// <param name="maxEventSubscribersPerSession">Maximum concurrent event subscribers allowed per session.</param>
public MultiSubscriberGatewayServiceFixture(
IWorkerProcessLauncher launcher,
int maxEventSubscribersPerSession = 8)
int maxEventSubscribersPerSession = 8,
Action<string>? diagnosticOutput = null)
{
_diagnostics = ((IDiagnosticFakeWorker)launcher).Diagnostics;
_diagnostics.SetOutput(diagnosticOutput);
IOptions<GatewayOptions> options = Options.Create(CreateOptions(maxEventSubscribersPerSession));
SessionWorkerClientFactory workerClientFactory = new(
launcher,
options,
_metrics,
NullLoggerFactory.Instance);
_diagnostics);
SessionManager sessionManager = new(
_registry,
workerClientFactory,
@@ -499,13 +505,20 @@ public sealed class GatewayEndToEndMultiSubscriberTests
/// <summary>Disposes every session in the registry and releases the fixture's metrics.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
// Session disposal and worker observation share the existing test budget.
// The observer consumes a late cleanup fault without replacing the primary failure.
_diagnostics.Track(DisposeSessionsAsync(), "teardown");
await _diagnostics.DrainAsync(TestTimeout).ConfigureAwait(false);
_metrics.Dispose();
}
private async Task DisposeSessionsAsync()
{
foreach (GatewaySession session in _registry.Snapshot())
{
await session.DisposeAsync();
await session.DisposeAsync().ConfigureAwait(false);
}
_metrics.Dispose();
}
private static GatewayOptions CreateOptions(int maxEventSubscribersPerSession) =>
@@ -540,7 +553,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
/// command, then waits for shutdown. Events carry an indexed string value so the test can
/// verify fan-out order across two subscribers.
/// </summary>
private sealed class MultiEventFakeWorkerProcessLauncher(int eventCount) : IWorkerProcessLauncher
private sealed class MultiEventFakeWorkerProcessLauncher(int eventCount) : IWorkerProcessLauncher, IDiagnosticFakeWorker
{
public const int ProcessId = 7710;
@@ -549,12 +562,16 @@ public sealed class GatewayEndToEndMultiSubscriberTests
/// <summary>Gets the task representing the fake worker's running background loop.</summary>
public Task WorkerTask { get; private set; } = Task.CompletedTask;
/// <inheritdoc />
public FakeWorkerDiagnostics Diagnostics { get; } = new();
/// <inheritdoc />
public Task<WorkerProcessHandle> LaunchAsync(
WorkerProcessLaunchRequest request,
CancellationToken cancellationToken = default)
{
WorkerTask = RunWorkerAsync(request, cancellationToken);
Diagnostics.Track(WorkerTask);
return Task.FromResult(new WorkerProcessHandle(
_process,
@@ -578,6 +595,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
{
WorkerEnvelope envelope = await harness.ReadGatewayEnvelopeAsync(cancellationToken).ConfigureAwait(false);
Diagnostics.Received(envelope);
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerShutdown)
{
await harness.SendShutdownAckAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
@@ -630,7 +648,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
/// deterministic. Call <see cref="StopEmitting"/> before closing the session so the
/// worker loop exits cleanly and can process the shutdown envelope.
/// </summary>
private sealed class GatedEventFakeWorkerProcessLauncher : IWorkerProcessLauncher
private sealed class GatedEventFakeWorkerProcessLauncher : IWorkerProcessLauncher, IDiagnosticFakeWorker
{
public const int ProcessId = 7720;
@@ -643,6 +661,9 @@ public sealed class GatewayEndToEndMultiSubscriberTests
/// <summary>Gets the task representing the fake worker's running background loop.</summary>
public Task WorkerTask { get; private set; } = Task.CompletedTask;
/// <inheritdoc />
public FakeWorkerDiagnostics Diagnostics { get; } = new();
/// <summary>Releases the gate so the worker emits one event.</summary>
public void AllowNextEvent() => _emitGate.Release();
@@ -662,6 +683,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
CancellationToken cancellationToken = default)
{
WorkerTask = RunWorkerAsync(request, cancellationToken);
Diagnostics.Track(WorkerTask);
return Task.FromResult(new WorkerProcessHandle(
_process,
@@ -717,6 +739,7 @@ public sealed class GatewayEndToEndMultiSubscriberTests
continue;
}
Diagnostics.Received(envelope);
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerShutdown)
{
await harness.SendShutdownAckAsync(cancellationToken: cancellationToken).ConfigureAwait(false);