Merge fix/c3-session-lifecycle: GWC-32 WorkerClient dispose is terminal, GWC-34 attach-after-dispose throws SessionNotReady, GWC-37 subtag watch-list coverage graded on provider status + re-resolved at failover edge

This commit is contained in:
Joseph Doherty
2026-08-20 09:39:53 -04:00
21 changed files with 864 additions and 50 deletions
@@ -384,9 +384,14 @@ public sealed class AlarmFailoverEndToEndTests
private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver
{
/// <inheritdoc />
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
public Task<AlarmWatchListResolution> ResolveAsync(
AlarmsOptions options,
CancellationToken cancellationToken = default) => Task.FromResult(targets);
CancellationToken cancellationToken = default) =>
Task.FromResult(new AlarmWatchListResolution(
targets,
DiscoveryAttempted: false,
DiscoveryFailed: false,
DiscoveryFailureMessage: null));
}
/// <summary>
@@ -507,10 +507,14 @@ public sealed class AlarmTruncationSignalTests
private sealed class StubWatchListResolver : IAlarmWatchListResolver
{
/// <inheritdoc />
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
public Task<AlarmWatchListResolution> ResolveAsync(
AlarmsOptions options,
CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyList<AlarmSubtagTarget>>([]);
Task.FromResult(new AlarmWatchListResolution(
[],
DiscoveryAttempted: false,
DiscoveryFailed: false,
DiscoveryFailureMessage: null));
}
/// <summary>
@@ -16,6 +16,21 @@ public sealed class AlarmWatchListResolverTests
private static AlarmWatchListResolver CreateResolver(IGalaxyRepository repository) =>
new(repository, NullLogger<AlarmWatchListResolver>.Instance);
/// <summary>
/// Resolves and projects to the target list. These cases assert composition rules, which
/// live in <see cref="AlarmWatchListResolution.Targets"/>; the discovery-provenance fields
/// the resolution also carries (GWC-37) are asserted by their own cases below.
/// </summary>
/// <param name="resolver">The resolver under test.</param>
/// <param name="options">Alarm options to resolve against.</param>
/// <param name="cancellationToken">Token to cancel the resolution.</param>
/// <returns>The resolved targets.</returns>
private static async Task<IReadOnlyList<AlarmSubtagTarget>> ResolveTargetsAsync(
AlarmWatchListResolver resolver,
AlarmsOptions options,
CancellationToken cancellationToken = default) =>
(await resolver.ResolveAsync(options, cancellationToken)).Targets;
private static AlarmsOptions Options(
bool useGalaxyRepository = true,
string area = "",
@@ -58,7 +73,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
include: ["pump01.fault", "Valve03.Position.Lo"],
exclude: ["Tank02.Level.HiHi"]));
@@ -81,7 +96,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
subtags: new AlarmSubtagNameOptions
{
Active = "InAlarm",
@@ -110,7 +125,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
// Default Priority is "Priority"; force it empty alongside empty AckComment.
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
subtags: new AlarmSubtagNameOptions { Priority = string.Empty, AckComment = string.Empty }));
AlarmSubtagTarget target = Assert.Single(result);
@@ -140,7 +155,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
// Config area "DEV" must be ignored for a GR row that has a discovered area.
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver,
Options(area: "DEV", defaultArea: "DEV"));
AlarmSubtagTarget target = Assert.Single(result);
@@ -161,7 +176,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
// No discovery area and no default area.
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options());
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options());
AlarmSubtagTarget target = Assert.Single(result);
Assert.Equal("Galaxy!Tank01.Level.HiHi", target.AlarmFullReference);
@@ -178,7 +193,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
area: "Site_A",
include: ["Tank01.Level.HiHi"]));
@@ -196,7 +211,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
area: "",
defaultArea: "Plant",
include: ["Tank01.Level.HiHi"]));
@@ -217,7 +232,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
useGalaxyRepository: false,
include: ["Tank01.Level.HiHi"]));
@@ -235,7 +250,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
include: ["Tank01.Level.HiHi"]));
// Did not throw; discovery set empty, include retained.
@@ -243,6 +258,51 @@ public sealed class AlarmWatchListResolverTests
Assert.Equal("Galaxy!Tank01.Level.HiHi", target.AlarmFullReference);
}
/// <summary>
/// GWC-37: swallowing the discovery failure is right for the resolver but leaves the
/// caller unable to tell "discovery produced nothing" from "nothing was configured", which
/// is why the empty watch-list could sit unnoticed for a monitor's whole life. The
/// resolution therefore reports whether discovery was attempted, whether it failed, and
/// the failure message the provider status quotes.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ResolveAsync_RepositoryThrows_ReportsDiscoveryFailureOnTheResolution()
{
ThrowingGalaxyRepository repo = new(new InvalidOperationException("SQL unavailable"));
AlarmWatchListResolver resolver = CreateResolver(repo);
AlarmWatchListResolution resolution = await resolver.ResolveAsync(Options());
Assert.Empty(resolution.Targets);
Assert.True(resolution.DiscoveryAttempted);
Assert.True(resolution.DiscoveryFailed);
Assert.Equal("SQL unavailable", resolution.DiscoveryFailureMessage);
}
/// <summary>
/// The complement of the case above: discovery disabled is not discovery failed. An empty
/// watch-list here means the deployment did not ask for subtag coverage, which the monitor
/// must not report as a degraded standby (GWC-37).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ResolveAsync_RepositoryDisabled_ReportsDiscoveryNotAttempted()
{
StubGalaxyRepository repo = new([]);
AlarmWatchListResolver resolver = CreateResolver(repo);
AlarmWatchListResolution resolution = await resolver.ResolveAsync(
Options(useGalaxyRepository: false));
Assert.Empty(resolution.Targets);
Assert.False(resolution.DiscoveryAttempted);
Assert.False(resolution.DiscoveryFailed);
Assert.Null(resolution.DiscoveryFailureMessage);
}
/// <summary>Verifies the source object reference is derived from a config include entry, using the portion before the first dot when present.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -252,7 +312,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
include: ["Tank01.Level.HiHi", "StandaloneTag"]));
Assert.Equal("Tank01", result[0].SourceObjectReference);
@@ -273,7 +333,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
useGalaxyRepository: false,
include: ["Tank01.Level.HiHi"],
exclude: ["Tank01.Level.HiHi"]));
@@ -299,7 +359,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
useGalaxyRepository: true,
exclude: ["Tank02.Level.HiHi"]));
@@ -324,7 +384,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
// The exclude array contains a whitespace-only string — should be a no-op.
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
useGalaxyRepository: true,
exclude: [" "]));
@@ -351,7 +411,7 @@ public sealed class AlarmWatchListResolverTests
AlarmWatchListResolver resolver = CreateResolver(repo);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
resolver.ResolveAsync(
ResolveTargetsAsync(resolver,
Options(include: ["Tank01.Level.HiHi"]),
cts.Token));
}
@@ -375,7 +435,7 @@ public sealed class AlarmWatchListResolverTests
// Pump01.Fault is an explicit include AND an exclude (case-insensitive). It must
// be dropped; the GR row Tank01 survives.
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
useGalaxyRepository: true,
include: ["Pump01.Fault"],
exclude: ["pump01.fault"]));
@@ -498,10 +498,14 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
private sealed class StubWatchListResolver : IAlarmWatchListResolver
{
/// <inheritdoc />
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
public Task<AlarmWatchListResolution> ResolveAsync(
AlarmsOptions options,
CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyList<AlarmSubtagTarget>>([]);
Task.FromResult(new AlarmWatchListResolution(
[],
DiscoveryAttempted: false,
DiscoveryFailed: false,
DiscoveryFailureMessage: null));
}
/// <summary>
@@ -623,6 +623,184 @@ public sealed class GatewayAlarmMonitorProviderModeTests
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// GWC-37: a Galaxy Repository outage at startup used to leave one <c>LogWarning</c> and an
/// empty subtag watch-list that stayed empty for the monitor's whole life — invisible until
/// a failover found the standby covering nothing. The condition is now a standing verdict
/// on the provider status, so it reaches the alarm feed, the dashboard badge and every
/// late-joining subscriber. The mode stays <c>ALARMMGR</c>: the live provider is healthy,
/// it is the standby behind it that is unsound.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubtagCoverage_WhenDiscoveryFailsAtSubscribe_ProviderStatusIsDegradedWithReason()
{
using GatewayMetrics metrics = new();
FakeSessionManager sessions = new();
ScriptedWatchListResolver resolver = new(DiscoveryFailedResolution());
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForSubscribeAsync(WaitTimeout);
// The empty list still went to the worker — the verdict is a report, not a behaviour change.
Assert.Empty(sessions.LastSubscribeCommand!.WatchList);
AlarmProviderStatus status = await ReadFirstProviderStatusAsync(monitor);
Assert.Equal(AlarmProviderMode.Alarmmgr, status.Mode);
Assert.True(status.Degraded);
Assert.Contains(AlarmProviderReasons.SubtagWatchListEmpty, status.Reason, StringComparison.Ordinal);
Assert.Contains(AlarmProviderReasons.SubtagDiscoveryFailed, status.Reason, StringComparison.Ordinal);
Assert.Contains("gr offline", status.Reason, StringComparison.Ordinal);
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// GWC-37: the failover edge is the moment the watch-list stops being theoretical, so the
/// monitor re-resolves it there and publishes the fresh verdict on the same status frame
/// the failover produces. Discovery that has recovered since <c>SubscribeAlarms</c> is
/// reported as drift rather than applied — the worker rejects a second <c>SubscribeAlarms</c>
/// while subscribed, and the only way to replace the list rebuilds the failover consumer on
/// the primary that just failed. The status names both the fresh count and the sent count so
/// an operator knows a monitor restart will pick the new list up.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubtagCoverage_WhenDiscoveryRecoversByFailover_ReResolvesAndReportsDrift()
{
using GatewayMetrics metrics = new();
FakeSessionManager sessions = new();
ScriptedWatchListResolver resolver = new(
DiscoveryFailedResolution(),
new AlarmWatchListResolution(
[Target("Galaxy!Area.Tank01.Level.HiHi"), Target("Galaxy!Area.Tank02.Level.HiHi")],
DiscoveryAttempted: true,
DiscoveryFailed: false,
DiscoveryFailureMessage: null));
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForSubscribeAsync(WaitTimeout);
Assert.Equal(1, resolver.Calls);
List<AlarmFeedMessage> received = [];
TaskCompletionSource baselineReceived = new(TaskCreationOptions.RunContinuationsAsynchronously);
using CancellationTokenSource streamCts = new();
Task reader = Task.Run(async () =>
{
try
{
await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, streamCts.Token))
{
lock (received)
{
received.Add(message);
if (received.Count == 1)
{
baselineReceived.TrySetResult();
}
}
}
}
catch (OperationCanceledException)
{
// Expected when the test cancels the stream.
}
});
await baselineReceived.Task.WaitAsync(WaitTimeout);
sessions.EmitEvent(new MxEvent
{
OnAlarmProviderModeChanged = new OnAlarmProviderModeChangedEvent
{
Mode = AlarmProviderMode.Subtag,
Reason = "alarmmgr failed",
At = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
},
});
AlarmFeedMessage degraded = await WaitForAsync(
received,
m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus
&& m.ProviderStatus.Mode == AlarmProviderMode.Subtag,
WaitTimeout);
// The re-resolve happened on the failover edge, before the status was published.
Assert.Equal(2, resolver.Calls);
string reason = degraded.ProviderStatus.Reason;
Assert.Contains("alarmmgr failed", reason, StringComparison.Ordinal);
Assert.Contains(AlarmProviderReasons.SubtagWatchListStale, reason, StringComparison.Ordinal);
Assert.Contains("2 target(s) now resolve against 0 sent", reason, StringComparison.Ordinal);
// The stale-list warning replaced the empty/failed pair: discovery works again, so the only
// remaining problem is that the worker is still using the list it was handed.
Assert.DoesNotContain(AlarmProviderReasons.SubtagWatchListEmpty, reason, StringComparison.Ordinal);
Assert.DoesNotContain(AlarmProviderReasons.SubtagDiscoveryFailed, reason, StringComparison.Ordinal);
await streamCts.CancelAsync();
await reader;
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>Reads the provider-status frame every stream opens with, then closes the stream.</summary>
/// <param name="monitor">The monitor to read from.</param>
/// <returns>The provider status carried by the first frame.</returns>
private static async Task<AlarmProviderStatus> ReadFirstProviderStatusAsync(GatewayAlarmMonitor monitor)
{
using CancellationTokenSource streamCts = new(WaitTimeout);
await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, streamCts.Token))
{
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus)
{
return message.ProviderStatus;
}
}
throw new TimeoutException("No provider-status message was received in time.");
}
/// <summary>A resolution standing for "Galaxy Repository discovery was on and it failed".</summary>
/// <returns>The resolution.</returns>
private static AlarmWatchListResolution DiscoveryFailedResolution() => new(
[],
DiscoveryAttempted: true,
DiscoveryFailed: true,
DiscoveryFailureMessage: "gr offline");
private static AlarmSubtagTarget Target(string fullReference) => new()
{
AlarmFullReference = fullReference,
SourceObjectReference = fullReference,
ActiveSubtag = $"{fullReference}.InAlarm",
AckedSubtag = $"{fullReference}.Acked",
};
private static GatewayAlarmMonitor CreateMonitor(
FakeSessionManager sessions,
GatewayMetrics metrics,
IAlarmWatchListResolver resolver)
{
AlarmsOptions options = new()
{
Enabled = true,
SubscriptionExpression = @"\\NODE\Galaxy!Area",
};
return new GatewayAlarmMonitor(
sessions,
resolver,
metrics,
Microsoft.Extensions.Options.Options.Create(new GatewayOptions { Alarms = options }),
NullLogger<GatewayAlarmMonitor>.Instance);
}
private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics)
{
AlarmsOptions options = new()
@@ -693,13 +871,42 @@ public sealed class GatewayAlarmMonitorProviderModeTests
throw new TimeoutException("Condition was not met in time.");
}
/// <summary>
/// <see cref="IAlarmWatchListResolver"/> that hands out a scripted sequence of
/// resolutions — one per call, the last repeating — and counts the calls, so a test can
/// drive "discovery was down at subscribe and back up by the failover" and assert that the
/// second resolution happened at all (GWC-37).
/// </summary>
/// <param name="script">Resolutions to return, in call order.</param>
private sealed class ScriptedWatchListResolver(params AlarmWatchListResolution[] script) : IAlarmWatchListResolver
{
private int _calls;
/// <summary>Gets the number of times <see cref="ResolveAsync"/> has been called.</summary>
public int Calls => Volatile.Read(ref _calls);
/// <inheritdoc />
public Task<AlarmWatchListResolution> ResolveAsync(
AlarmsOptions options,
CancellationToken cancellationToken = default)
{
int index = Interlocked.Increment(ref _calls) - 1;
return Task.FromResult(script[Math.Min(index, script.Length - 1)]);
}
}
/// <summary><see cref="IAlarmWatchListResolver"/> that returns a fixed watch-list.</summary>
private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver
{
/// <inheritdoc />
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
public Task<AlarmWatchListResolution> ResolveAsync(
AlarmsOptions options,
CancellationToken cancellationToken = default) => Task.FromResult(targets);
CancellationToken cancellationToken = default) =>
Task.FromResult(new AlarmWatchListResolution(
targets,
DiscoveryAttempted: false,
DiscoveryFailed: false,
DiscoveryFailureMessage: null));
}
/// <summary>
@@ -179,8 +179,10 @@ public sealed class DashboardBrowseAndAlarmModelTests
/// <summary>
/// An explicitly-degraded status whose mode is still Alarmmgr (the
/// <c>Degraded || Mode==Subtag</c> guard's second, independent branch) must still
/// map to the degraded amber badge.
/// <c>Degraded || Mode==Subtag</c> guard's second, independent branch) must map to the
/// amber badge, but not to the failover label: GWC-37 gives that combination a meaning —
/// the alarm manager is serving and the subtag standby behind it covers nothing — so
/// reporting "Subtag monitoring (degraded)" would name the wrong live provider.
/// </summary>
[Fact]
public void FromProviderStatus_Alarmmgr_DegradedFlagSet_WarningBadge()
@@ -195,8 +197,9 @@ public sealed class DashboardBrowseAndAlarmModelTests
DashboardAlarmProviderStatus model = DashboardAlarmProviderStatus.FromProviderStatus(status);
Assert.True(model.IsDegraded);
Assert.Equal(DashboardAlarmProviderStatus.DegradedLabel, model.Label);
Assert.Equal(DashboardAlarmProviderStatus.NoSubtagStandbyLabel, model.Label);
Assert.Contains("bg-warning", model.BadgeCssClass, StringComparison.Ordinal);
Assert.Equal(AlarmProviderMode.Alarmmgr, model.Mode);
}
/// <summary>
@@ -719,6 +719,47 @@ public sealed class GatewaySessionTests
Assert.Equal([1UL, 2UL], received.Select(mxEvent => mxEvent.WorkerSequence).ToArray());
}
/// <summary>
/// GWC-34, the disposal-side twin of the GWC-27 premature-attach poison: an attach that
/// passed its readiness gate and then lost the CPU until the session was torn down used to
/// find <c>_eventDistributor</c> null, read that as "not created yet" and build a
/// REPLACEMENT distributor. Its pump could never start (the started flag is a one-way latch)
/// and its channel was never completed, so the subscriber registered on it and blocked in
/// <c>WaitToReadAsync</c> until its own deadline — no error, no log, no metric. Disposal now
/// latches the null-out, so every attach entry point fails immediately and loudly instead.
/// Attaching twice pins the rollback: the second call must still report
/// <c>SessionNotReady</c>, not <c>EventSubscriberAlreadyActive</c>, which it only can if the
/// first failure released the subscriber slot it took.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AttachAfterDisposeThrowsSessionNotReadyInsteadOfRegisteringOnANeverPumpedDistributor()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
// Disposal leaves the session state Ready (only CloseAsync transitions it), so the attach
// paths' readiness gates pass and the calls below genuinely reach EnsureDistributorCreated —
// the exact position the racing attach in the finding occupies.
await session.DisposeAsync();
SessionManagerException attachException = Assert.Throws<SessionManagerException>(
() => session.AttachEventSubscriber(maxSubscribers: 1));
Assert.Equal(SessionManagerErrorCode.SessionNotReady, attachException.ErrorCode);
SessionManagerException secondAttachException = Assert.Throws<SessionManagerException>(
() => session.AttachEventSubscriber(maxSubscribers: 1));
Assert.Equal(SessionManagerErrorCode.SessionNotReady, secondAttachException.ErrorCode);
SessionManagerException replayException = Assert.Throws<SessionManagerException>(
() => session.AttachEventSubscriberWithReplay(maxSubscribers: 1, afterSequence: 0));
Assert.Equal(SessionManagerErrorCode.SessionNotReady, replayException.ErrorCode);
SessionManagerException internalException = Assert.Throws<SessionManagerException>(
() => session.AttachInternalEventSubscriber());
Assert.Equal(SessionManagerErrorCode.SessionNotReady, internalException.ErrorCode);
}
private static GatewaySession CreateSession()
{
return new GatewaySession(
@@ -14,6 +14,10 @@ public sealed class WorkerClientTests
private const string SessionId = "session-worker-client";
private const string Nonce = "nonce-worker-client";
private const int WorkerProcessId = 4321;
/// <summary>Upper bound on the events <see cref="EmitEventsUntilStoppedAsync"/> writes.</summary>
private const ulong EmittedEventCeiling = 2000;
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
/// <summary>Verifies that StartAsync enters ready state after receiving worker hello and ready messages.</summary>
@@ -705,6 +709,96 @@ public sealed class WorkerClientTests
Assert.True(process.Disposed);
}
/// <summary>
/// GWC-32: disposing a still-<c>Ready</c> client while the worker is emitting events must be
/// an ordinary teardown, not a fault. <c>DisposeAsync</c> completes the same three channels
/// <c>MarkClosed</c> and <c>SetFaulted</c> do, so it must enter a terminal state first;
/// otherwise the read loop — already inside <c>DispatchEnvelope</c> when the staging channel
/// is completed, or reading a stream that is torn down underneath it — misdiagnoses the
/// completed channel as the GWC-24 sustained-backpressure overflow and raises a spurious
/// <c>ProtocolViolation</c> fault, a <c>worker-event-staging</c> overflow metric and a
/// redundant kill during an intentional dispose. The event channel is sized far above the
/// burst so a genuine overflow is impossible, and <c>DisposeAsync</c> awaits its background
/// tasks, so the metrics read afterwards is not racing a late fault.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DisposeAsync_WhileWorkerIsEmittingEvents_ClosesWithoutFaultingOrOverflowing()
{
// The burst is capped at EmittedEventCeiling and the staging channel holds 2 x capacity, so
// even if every emitted event were staged and none consumed the GWC-24 bound is unreachable:
// an overflow metric in this test can only be the dispose-race misdiagnosis.
const int capacity = 2048;
using GatewayMetrics metrics = new();
await using PipePair pipePair = await PipePair.CreateAsync();
WorkerClient client = CreateClient(
pipePair,
new WorkerClientOptions
{
EventChannelCapacity = capacity,
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
HeartbeatGrace = TimeSpan.FromSeconds(30),
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
},
metrics: metrics);
await CompleteHandshakeAsync(client, pipePair);
using CancellationTokenSource emitCts = new(TestTimeout);
Task emitTask = EmitEventsUntilStoppedAsync(pipePair, emitCts.Token);
// Proves the read loop is mid-stream — not parked before its first event — when dispose runs.
await WaitUntilAsync(() => metrics.GetSnapshot().WorkerEventQueueDepth > 0, TestTimeout);
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
await emitCts.CancelAsync();
await emitTask.WaitAsync(TestTimeout);
Assert.Equal(WorkerClientState.Closed, client.State);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(0L, snapshot.Faults);
Assert.Equal(0L, snapshot.QueueOverflows);
Assert.Equal(0L, snapshot.WorkerKills);
// Disposal is now a terminal transition, so the worker that reached Ready is recorded as
// stopped exactly once — the running-worker gauge no longer leaks on this path.
Assert.Equal(0, snapshot.WorkersRunning);
Assert.Equal(1L, snapshot.WorkerExits);
}
/// <summary>
/// Writes worker events from the fake worker side until cancelled or until
/// <see cref="EmittedEventCeiling"/> events have been written, so the gateway read loop has a
/// continuous stream to dispatch while a test tears the client down. The ceiling keeps the
/// burst structurally below the staging bound the caller configures. Every failure is
/// swallowed: disposal closes the gateway end of the pipe underneath this writer, which is
/// the situation under test rather than a test failure.
/// </summary>
/// <param name="pipePair">The connected pipe pair whose worker side emits the events.</param>
/// <param name="cancellationToken">Stops the emission.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private static async Task EmitEventsUntilStoppedAsync(
PipePair pipePair,
CancellationToken cancellationToken)
{
await Task.Yield();
try
{
for (ulong sequence = 1;
sequence <= EmittedEventCeiling && !cancellationToken.IsCancellationRequested;
sequence++)
{
await pipePair.WorkerWriter.WriteAsync(
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange),
cancellationToken);
}
}
catch (Exception)
{
// Broken pipe, disposed stream or cancellation — all expected once dispose starts.
}
}
/// <summary>
/// Verifies that a heartbeat envelope updates the last-heartbeat timestamp and worker
/// process id. Uses a <see cref="ManualTimeProvider"/> so the timestamp advance is