Merge branch 'fix/gwc-26-27-alarm-attach'
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m17s
ci / java (push) Successful in 2m4s
ci / portable (push) Successful in 7m8s

# Conflicts:
#	archreview/2026-07-12/remediation/00-tracking.md
#	archreview/2026-07-12/remediation/10-gateway-core.md
This commit is contained in:
Joseph Doherty
2026-08-07 06:02:17 -04:00
19 changed files with 738 additions and 130 deletions
@@ -210,6 +210,17 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
try
{
// Attach the internal distributor subscriber BEFORE subscribing (GWC-26). The pump
// has been running since MarkReady started the dashboard mirror, and the distributor
// only fans to subscribers registered at the time of the fan-out, so a subscriber
// taken after SubscribeAlarms + the first reconcile would silently lose every
// transition raised inside that two-round-trip window — and a missed Acknowledge is
// never repaired by the presence-only reconcile deltas. Transitions arriving while we
// subscribe and reconcile simply buffer in this lease's bounded channel; if it ever
// overflowed, the internal subscriber is disconnected (it never faults the session),
// the enumeration below ends, and the supervisor loop restarts the lifecycle.
using IEventSubscriberLease alarmLease = session.AttachInternalEventSubscriber();
await SubscribeAlarmsAsync(session.SessionId, subscription, stoppingToken).ConfigureAwait(false);
await ReconcileAsync(session.SessionId, stoppingToken).ConfigureAwait(false);
@@ -228,9 +239,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
// Consume mapped MxEvents through the session's single distributor pump (as an
// internal, non-counted subscriber) rather than opening a second raw drain of the
// worker event channel — a second drain would split events with the dashboard
// mirror pump and silently lose Acknowledge/mode-change transitions.
await foreach (MxEvent mxEvent in _sessionManager
.ReadAlarmEventsAsync(session.SessionId, linked.Token)
// mirror pump and silently lose Acknowledge/mode-change transitions. The lease was
// taken above, before SubscribeAlarms; draining it only now is order-safe because
// ApplyTransition handles alarms the reconcile already placed in the cache.
await foreach (MxEvent mxEvent in alarmLease.Reader
.ReadAllAsync(linked.Token)
.ConfigureAwait(false))
{
if (mxEvent is { BodyCase: MxEvent.BodyOneofCase.OnAlarmTransition }
@@ -511,6 +524,20 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
// Replaces the cache with the worker's authoritative snapshot, broadcasting
// a synthetic transition for any alarm the live stream missed.
//
// Repair scope (GWC-26): presence deltas (Clear/Raise) plus acked-state deltas. These are
// ALARM FEED transitions (AlarmFeedMessage on the StreamAlarms/dashboard surface), rebuilt
// from the worker's own authoritative snapshot to repair what the live feed missed. They are
// not MxEvents and never reach the gRPC StreamEvents path, so this feed-level repair does not
// breach the "never synthesize events" rule, which governs MxEvent emission.
//
// Delivery semantics: feed repair transitions are AT-LEAST-ONCE, not exactly-once. A reconcile
// reads the worker's current state while the corresponding live transition may still be
// buffered in the alarm lease's channel; both then broadcast, and the two are indistinguishable
// on the feed. This is inherent to the reconcile design and pre-dates the acked-state delta
// (the Raise/Clear repair has always had it), since nothing serializes a reconcile against the
// in-flight live stream. Consumers must therefore treat alarm state idempotently — apply a
// transition as "set the alarm to this state", never as an increment or a toggle.
private void ApplyReconcile(IEnumerable<ActiveAlarmSnapshot> snapshots)
{
Dictionary<string, ActiveAlarmSnapshot> next = new(StringComparer.Ordinal);
@@ -536,12 +563,23 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
foreach (KeyValuePair<string, ActiveAlarmSnapshot> incoming in next)
{
if (!_alarms.ContainsKey(incoming.Key))
if (!_alarms.TryGetValue(incoming.Key, out ActiveAlarmSnapshot? existing))
{
Broadcast(
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Raise) },
incoming.Key);
}
else if (existing.CurrentState != incoming.Value.CurrentState
&& incoming.Value.CurrentState == AlarmConditionState.ActiveAcked)
{
// The alarm was already known but the worker now reports it acknowledged: the
// live Acknowledge transition never reached the feed. Without this the acked
// state is absorbed silently by the snapshot replace below and subscribers show
// the alarm unacked until it clears.
Broadcast(
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Acknowledge) },
incoming.Key);
}
}
_alarms.Clear();
@@ -548,10 +548,37 @@ public sealed class GatewaySession
/// <c>MaxEventSubscribersPerSession</c> accounting and out of the single-subscriber
/// overflow-fault path, so a slow alarm reconcile can never fault the session — it only
/// disconnects this internal subscriber.
/// <para>
/// Gated on readiness exactly like <see cref="AttachEventSubscriber"/>: attaching
/// before the session and its worker are <c>Ready</c> throws
/// <see cref="SessionManagerException"/> with
/// <see cref="SessionManagerErrorCode.SessionNotReady"/>.
/// </para>
/// </remarks>
/// <returns>The internal subscriber's lease; dispose it to unregister.</returns>
/// <exception cref="SessionManagerException">
/// The session or its worker client is not <c>Ready</c>.
/// </exception>
public IEventSubscriberLease AttachInternalEventSubscriber()
{
// Readiness gate, mirroring AttachEventSubscriber (GWC-27). It must run BEFORE
// EnsureDistributorCreated: a premature attach would construct the distributor and start
// its pump against a not-yet-Ready worker, the pump source would throw SessionNotReady,
// PumpAsync would complete every subscriber with that error and latch _completed, and
// _eventDistributorStarted is never reset — so the session would reach Ready with
// permanently dead event streaming, silently, for the rest of its life. Failing loudly
// here keeps that state unreachable. The check is under _syncRoot and the distributor
// calls stay outside it, matching AttachEventSubscriber's lock discipline.
lock (_syncRoot)
{
if (_state != SessionState.Ready || _workerClient?.State != WorkerClientState.Ready)
{
throw new SessionManagerException(
SessionManagerErrorCode.SessionNotReady,
$"Session {SessionId} is not ready for event streaming. Current state is {_state}.");
}
}
// Same sequence StartDashboardMirror uses: create the distributor (claiming the pump
// start if we are first), register the internal subscriber BEFORE the pump starts so a
// subscriber is always present at pump start, then start the pump if requested.
@@ -43,18 +43,6 @@ public interface ISessionManager
string sessionId,
CancellationToken cancellationToken);
/// <summary>
/// Reads mapped events for the central alarm monitor by attaching an internal
/// (non-counted) distributor subscriber, so the alarm feed shares the one worker-event
/// pump instead of opening a second raw drain of the single worker event channel.
/// </summary>
/// <param name="sessionId">Identifier of the session.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>The mapped <see cref="MxEvent"/>s fanned by the session's distributor.</returns>
IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
string sessionId,
CancellationToken cancellationToken);
/// <summary>Closes a session and terminates its worker process.</summary>
/// <param name="sessionId">Identifier of the session to close.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
@@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging;
@@ -191,22 +190,6 @@ public sealed class SessionManager : ISessionManager
return session.ReadEventsAsync(cancellationToken);
}
/// <inheritdoc />
public async IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
string sessionId,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
GatewaySession session = GetRequiredSession(sessionId);
using IEventSubscriberLease lease = session.AttachInternalEventSubscriber();
await foreach (MxEvent mxEvent in lease.Reader
.ReadAllAsync(cancellationToken)
.ConfigureAwait(false))
{
yield return mxEvent;
}
}
/// <inheritdoc />
public async Task<SessionCloseResult> CloseSessionAsync(
string sessionId,