fix(archreview): gateway core P0 remediation (GWC-01/02/03, TST-02, TST-12)
Interlocking changes across the gateway server (shared GatewaySession.cs / SessionManager.cs), committed together: - GWC-01 (Critical): alarm monitor now attaches as an internal (non-counted) distributor subscriber instead of a second raw drain of the single worker event channel; WorkerClient._events -> SingleReader with a claimed-once guard so a future dual-consumer regression throws loudly. - GWC-02 (High): faulted sessions are swept in CloseExpiredLeasesAsync (IsFaultedReapable + FaultedReason); new FaultedGraceSeconds (default 0). - GWC-03 (High): configurable MaxSparseArrayLength (default 1_000_000) enforced before allocation. - TST-02 (High, security): StreamEvents attach now enforces the opening key id -> PermissionDenied on owner mismatch. - TST-12 (Medium): CLAUDE.md retention-defaults sentence corrected. Verified: NonWindows build clean; targeted tests 135/135 on macOS, plus WorkerClientTests 18/18 on the Windows host.
This commit is contained in:
Joseph Doherty
+78
-1
@@ -88,7 +88,7 @@ public sealed class GatewaySessionDashboardMirrorTests
|
||||
Task grpcReader = Task.Run(async () =>
|
||||
{
|
||||
await foreach (MxEvent mxEvent in service
|
||||
.StreamEventsAsync(new StreamEventsRequest { SessionId = session.SessionId }, CancellationToken.None)
|
||||
.StreamEventsAsync(new StreamEventsRequest { SessionId = session.SessionId }, callerKeyId: null, CancellationToken.None)
|
||||
.WithCancellation(CancellationToken.None))
|
||||
{
|
||||
grpcEvents.Add(mxEvent);
|
||||
@@ -107,6 +107,65 @@ public sealed class GatewaySessionDashboardMirrorTests
|
||||
Assert.Equal([1UL, 2UL, 3UL], broadcaster.Captures.Select(capture => capture.MxEvent.WorkerSequence).ToArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-01 regression: with the internal dashboard mirror active, a second internal
|
||||
/// subscriber (the alarm monitor's feed, attached via
|
||||
/// <see cref="GatewaySession.AttachInternalEventSubscriber"/>) receives EVERY event —
|
||||
/// including the alarm <c>Acknowledge</c> transition — rather than the two consumers
|
||||
/// each getting a random half of the single worker channel. Before the fix the alarm
|
||||
/// monitor drained the worker channel directly, so with the dashboard mirror pump also
|
||||
/// draining it the two split the stream and Acknowledge transitions were silently lost.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task InternalAlarmSubscriber_AndDashboardMirror_BothReceiveEveryEvent()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
workerClient.Events.Add(CreateWorkerEvent(1, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateAlarmTransitionEvent(2, AlarmTransitionKind.Raise));
|
||||
workerClient.Events.Add(CreateAlarmTransitionEvent(3, AlarmTransitionKind.Acknowledge));
|
||||
workerClient.Events.Add(CreateAlarmTransitionEvent(4, AlarmTransitionKind.Clear));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
// Hold the finite stream until BOTH internal subscribers (dashboard mirror + alarm feed)
|
||||
// are registered so neither misses an event before the pump drains.
|
||||
workerClient.HoldEventsUntilReleased();
|
||||
RecordingDashboardEventBroadcaster broadcaster = new();
|
||||
|
||||
await using GatewaySession session = CreateSession(workerClient, broadcaster);
|
||||
session.AttachWorkerClient(workerClient);
|
||||
|
||||
// MarkReady registers the internal dashboard subscriber and starts the (gated) pump.
|
||||
session.MarkReady();
|
||||
|
||||
// Attach the alarm monitor's internal subscriber and drain it concurrently. Registered
|
||||
// BEFORE the stream is released, so it is present at pump start alongside the dashboard.
|
||||
using IEventSubscriberLease alarmLease = session.AttachInternalEventSubscriber();
|
||||
List<MxEvent> alarmEvents = [];
|
||||
Task alarmReader = Task.Run(async () =>
|
||||
{
|
||||
await foreach (MxEvent mxEvent in alarmLease.Reader.ReadAllAsync(CancellationToken.None))
|
||||
{
|
||||
alarmEvents.Add(mxEvent);
|
||||
}
|
||||
});
|
||||
|
||||
workerClient.ReleaseEvents();
|
||||
|
||||
await WaitUntilAsync(() => broadcaster.Captures.Count == 4 && alarmEvents.Count == 4);
|
||||
await alarmReader.WaitAsync(TestTimeout);
|
||||
|
||||
// Both internal consumers see the full, identical, ordered stream — no splitting.
|
||||
Assert.Equal([1UL, 2UL, 3UL, 4UL], broadcaster.Captures.Select(capture => capture.MxEvent.WorkerSequence).ToArray());
|
||||
Assert.Equal([1UL, 2UL, 3UL, 4UL], alarmEvents.Select(mxEvent => mxEvent.WorkerSequence).ToArray());
|
||||
|
||||
// The Acknowledge transition specifically reaches the alarm feed (the transition the
|
||||
// pre-fix split silently dropped, leaving clients showing unacked alarms indefinitely).
|
||||
Assert.Contains(
|
||||
alarmEvents,
|
||||
mxEvent => mxEvent.BodyCase == MxEvent.BodyOneofCase.OnAlarmTransition
|
||||
&& mxEvent.OnAlarmTransition.TransitionKind == AlarmTransitionKind.Acknowledge);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hazard guard: starting the pump at Ready with a fast-completing worker stream
|
||||
/// and zero subscribers used to drain into nothing and leave a later subscriber hanging.
|
||||
@@ -222,6 +281,19 @@ public sealed class GatewaySessionDashboardMirrorTests
|
||||
return new WorkerEvent { Event = mxEvent };
|
||||
}
|
||||
|
||||
private static WorkerEvent CreateAlarmTransitionEvent(ulong sequence, AlarmTransitionKind kind)
|
||||
{
|
||||
MxEvent mxEvent = new()
|
||||
{
|
||||
SessionId = "session-dashboard-mirror",
|
||||
Family = MxEventFamily.OnAlarmTransition,
|
||||
WorkerSequence = sequence,
|
||||
OnAlarmTransition = new OnAlarmTransitionEvent { TransitionKind = kind },
|
||||
};
|
||||
|
||||
return new WorkerEvent { Event = mxEvent };
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null)
|
||||
{
|
||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||||
@@ -280,6 +352,11 @@ public sealed class GatewaySessionDashboardMirrorTests
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken) => session.ReadEventsAsync(cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<SessionCloseResult> CloseSessionAsync(
|
||||
string sessionId,
|
||||
|
||||
@@ -975,6 +975,68 @@ public sealed class SessionManagerTests
|
||||
Assert.Equal(0, workerClient.ShutdownCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A faulted session is reaped by the lease sweep (default <c>FaultedGraceSeconds=0</c>)
|
||||
/// even though its normal lease is still far in the future, tearing down its worker and
|
||||
/// freeing the slot, while a healthy leased session in the same manager is untouched.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_ReapsFaultedSession()
|
||||
{
|
||||
FakeWorkerClient faultedClient = new();
|
||||
FakeWorkerClient healthyClient = new();
|
||||
QueueingSessionWorkerClientFactory factory = new(faultedClient, healthyClient);
|
||||
SessionManager manager = CreateManager(factory);
|
||||
GatewaySession faultedSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
GatewaySession healthySession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Both leases are far in the future, so only the fault can reap the faulted session.
|
||||
faultedSession.ExtendLease(now.AddMinutes(30));
|
||||
healthySession.ExtendLease(now.AddMinutes(30));
|
||||
faultedSession.MarkFaulted("test fault");
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedCount);
|
||||
Assert.Equal(SessionState.Closed, faultedSession.State);
|
||||
Assert.Equal(1, faultedClient.ShutdownCount);
|
||||
Assert.Equal(SessionState.Ready, healthySession.State);
|
||||
Assert.Equal(0, healthyClient.ShutdownCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// With a positive <c>FaultedGraceSeconds</c>, a faulted session stays observable until
|
||||
/// the grace window elapses, then the next sweep reaps it.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_RespectsFaultedGraceWindow()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(defaultLeaseSeconds: 1800, faultedGraceSeconds: 30),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
session.MarkFaulted("test fault");
|
||||
|
||||
// Before the grace window elapses: the faulted session is retained (still observable).
|
||||
clock.Advance(TimeSpan.FromSeconds(29));
|
||||
int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
Assert.Equal(0, closedBefore);
|
||||
Assert.Equal(SessionState.Faulted, session.State);
|
||||
|
||||
// After the grace window elapses: the sweep reaps it.
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
Assert.Equal(1, closedAfter);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -1023,7 +1085,8 @@ public sealed class SessionManagerTests
|
||||
int maxSessions = 64,
|
||||
int defaultLeaseSeconds = 1800,
|
||||
int detachGraceSeconds = 0,
|
||||
int workerReadyWaitTimeoutMs = 0)
|
||||
int workerReadyWaitTimeoutMs = 0,
|
||||
int faultedGraceSeconds = 0)
|
||||
{
|
||||
return new GatewayOptions
|
||||
{
|
||||
@@ -1034,6 +1097,7 @@ public sealed class SessionManagerTests
|
||||
DefaultLeaseSeconds = defaultLeaseSeconds,
|
||||
DetachGraceSeconds = detachGraceSeconds,
|
||||
WorkerReadyWaitTimeoutMs = workerReadyWaitTimeoutMs,
|
||||
FaultedGraceSeconds = faultedGraceSeconds,
|
||||
},
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
|
||||
@@ -222,4 +222,35 @@ public sealed class SparseArrayExpanderTests
|
||||
RpcException ex = Assert.Throws<RpcException>(() => SparseArrayExpander.Expand(value));
|
||||
Assert.Equal(StatusCode.InvalidArgument, ex.StatusCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a total length above the configured cap throws <see cref="StatusCode.InvalidArgument"/> before the full array is allocated.</summary>
|
||||
[Fact]
|
||||
public void Expand_TotalLengthExceedsConfiguredCap_ThrowsBeforeAllocation()
|
||||
{
|
||||
// A total_length that would force a huge allocation, but well below Array.MaxLength,
|
||||
// so only the configured cap can reject it (the Array.MaxLength backstop would not).
|
||||
MxValue value = SparseValue(MxDataType.Integer, 500_000_000u);
|
||||
|
||||
RpcException ex = Assert.Throws<RpcException>(() => SparseArrayExpander.Expand(value, maxSparseArrayLength: 1_000_000));
|
||||
Assert.Equal(StatusCode.InvalidArgument, ex.StatusCode);
|
||||
Assert.Contains("MaxSparseArrayLength", ex.Status.Detail, StringComparison.Ordinal);
|
||||
|
||||
// The value must be untouched — expansion (allocation) never ran.
|
||||
Assert.Equal(MxValue.KindOneofCase.SparseArrayValue, value.KindCase);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a total length at or below the configured cap still expands normally.</summary>
|
||||
[Fact]
|
||||
public void Expand_TotalLengthAtConfiguredCap_Expands()
|
||||
{
|
||||
MxValue value = SparseValue(
|
||||
MxDataType.Integer,
|
||||
4,
|
||||
(1, new MxValue { Int32Value = 7 }));
|
||||
|
||||
SparseArrayExpander.Expand(value, maxSparseArrayLength: 4);
|
||||
|
||||
Assert.Equal(MxValue.KindOneofCase.ArrayValue, value.KindCase);
|
||||
Assert.Equal(new[] { 0, 7, 0, 0 }, value.ArrayValue.Int32Values.Values);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user