Merge branch 'fix/gwc-26-27-alarm-attach'
# Conflicts: # archreview/2026-07-12/remediation/00-tracking.md # archreview/2026-07-12/remediation/10-gateway-core.md
This commit is contained in:
@@ -210,6 +210,17 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
|
||||
try
|
||||
{
|
||||
// Attach the internal distributor subscriber BEFORE subscribing (GWC-26). The pump
|
||||
// has been running since MarkReady started the dashboard mirror, and the distributor
|
||||
// only fans to subscribers registered at the time of the fan-out, so a subscriber
|
||||
// taken after SubscribeAlarms + the first reconcile would silently lose every
|
||||
// transition raised inside that two-round-trip window — and a missed Acknowledge is
|
||||
// never repaired by the presence-only reconcile deltas. Transitions arriving while we
|
||||
// subscribe and reconcile simply buffer in this lease's bounded channel; if it ever
|
||||
// overflowed, the internal subscriber is disconnected (it never faults the session),
|
||||
// the enumeration below ends, and the supervisor loop restarts the lifecycle.
|
||||
using IEventSubscriberLease alarmLease = session.AttachInternalEventSubscriber();
|
||||
|
||||
await SubscribeAlarmsAsync(session.SessionId, subscription, stoppingToken).ConfigureAwait(false);
|
||||
await ReconcileAsync(session.SessionId, stoppingToken).ConfigureAwait(false);
|
||||
|
||||
@@ -228,9 +239,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
// Consume mapped MxEvents through the session's single distributor pump (as an
|
||||
// internal, non-counted subscriber) rather than opening a second raw drain of the
|
||||
// worker event channel — a second drain would split events with the dashboard
|
||||
// mirror pump and silently lose Acknowledge/mode-change transitions.
|
||||
await foreach (MxEvent mxEvent in _sessionManager
|
||||
.ReadAlarmEventsAsync(session.SessionId, linked.Token)
|
||||
// mirror pump and silently lose Acknowledge/mode-change transitions. The lease was
|
||||
// taken above, before SubscribeAlarms; draining it only now is order-safe because
|
||||
// ApplyTransition handles alarms the reconcile already placed in the cache.
|
||||
await foreach (MxEvent mxEvent in alarmLease.Reader
|
||||
.ReadAllAsync(linked.Token)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
if (mxEvent is { BodyCase: MxEvent.BodyOneofCase.OnAlarmTransition }
|
||||
@@ -511,6 +524,20 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
|
||||
// Replaces the cache with the worker's authoritative snapshot, broadcasting
|
||||
// a synthetic transition for any alarm the live stream missed.
|
||||
//
|
||||
// Repair scope (GWC-26): presence deltas (Clear/Raise) plus acked-state deltas. These are
|
||||
// ALARM FEED transitions (AlarmFeedMessage on the StreamAlarms/dashboard surface), rebuilt
|
||||
// from the worker's own authoritative snapshot to repair what the live feed missed. They are
|
||||
// not MxEvents and never reach the gRPC StreamEvents path, so this feed-level repair does not
|
||||
// breach the "never synthesize events" rule, which governs MxEvent emission.
|
||||
//
|
||||
// Delivery semantics: feed repair transitions are AT-LEAST-ONCE, not exactly-once. A reconcile
|
||||
// reads the worker's current state while the corresponding live transition may still be
|
||||
// buffered in the alarm lease's channel; both then broadcast, and the two are indistinguishable
|
||||
// on the feed. This is inherent to the reconcile design and pre-dates the acked-state delta
|
||||
// (the Raise/Clear repair has always had it), since nothing serializes a reconcile against the
|
||||
// in-flight live stream. Consumers must therefore treat alarm state idempotently — apply a
|
||||
// transition as "set the alarm to this state", never as an increment or a toggle.
|
||||
private void ApplyReconcile(IEnumerable<ActiveAlarmSnapshot> snapshots)
|
||||
{
|
||||
Dictionary<string, ActiveAlarmSnapshot> next = new(StringComparer.Ordinal);
|
||||
@@ -536,12 +563,23 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
|
||||
foreach (KeyValuePair<string, ActiveAlarmSnapshot> incoming in next)
|
||||
{
|
||||
if (!_alarms.ContainsKey(incoming.Key))
|
||||
if (!_alarms.TryGetValue(incoming.Key, out ActiveAlarmSnapshot? existing))
|
||||
{
|
||||
Broadcast(
|
||||
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Raise) },
|
||||
incoming.Key);
|
||||
}
|
||||
else if (existing.CurrentState != incoming.Value.CurrentState
|
||||
&& incoming.Value.CurrentState == AlarmConditionState.ActiveAcked)
|
||||
{
|
||||
// The alarm was already known but the worker now reports it acknowledged: the
|
||||
// live Acknowledge transition never reached the feed. Without this the acked
|
||||
// state is absorbed silently by the snapshot replace below and subscribers show
|
||||
// the alarm unacked until it clears.
|
||||
Broadcast(
|
||||
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Acknowledge) },
|
||||
incoming.Key);
|
||||
}
|
||||
}
|
||||
|
||||
_alarms.Clear();
|
||||
|
||||
@@ -548,10 +548,37 @@ public sealed class GatewaySession
|
||||
/// <c>MaxEventSubscribersPerSession</c> accounting and out of the single-subscriber
|
||||
/// overflow-fault path, so a slow alarm reconcile can never fault the session — it only
|
||||
/// disconnects this internal subscriber.
|
||||
/// <para>
|
||||
/// Gated on readiness exactly like <see cref="AttachEventSubscriber"/>: attaching
|
||||
/// before the session and its worker are <c>Ready</c> throws
|
||||
/// <see cref="SessionManagerException"/> with
|
||||
/// <see cref="SessionManagerErrorCode.SessionNotReady"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <returns>The internal subscriber's lease; dispose it to unregister.</returns>
|
||||
/// <exception cref="SessionManagerException">
|
||||
/// The session or its worker client is not <c>Ready</c>.
|
||||
/// </exception>
|
||||
public IEventSubscriberLease AttachInternalEventSubscriber()
|
||||
{
|
||||
// Readiness gate, mirroring AttachEventSubscriber (GWC-27). It must run BEFORE
|
||||
// EnsureDistributorCreated: a premature attach would construct the distributor and start
|
||||
// its pump against a not-yet-Ready worker, the pump source would throw SessionNotReady,
|
||||
// PumpAsync would complete every subscriber with that error and latch _completed, and
|
||||
// _eventDistributorStarted is never reset — so the session would reach Ready with
|
||||
// permanently dead event streaming, silently, for the rest of its life. Failing loudly
|
||||
// here keeps that state unreachable. The check is under _syncRoot and the distributor
|
||||
// calls stay outside it, matching AttachEventSubscriber's lock discipline.
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_state != SessionState.Ready || _workerClient?.State != WorkerClientState.Ready)
|
||||
{
|
||||
throw new SessionManagerException(
|
||||
SessionManagerErrorCode.SessionNotReady,
|
||||
$"Session {SessionId} is not ready for event streaming. Current state is {_state}.");
|
||||
}
|
||||
}
|
||||
|
||||
// Same sequence StartDashboardMirror uses: create the distributor (claiming the pump
|
||||
// start if we are first), register the internal subscriber BEFORE the pump starts so a
|
||||
// subscriber is always present at pump start, then start the pump if requested.
|
||||
|
||||
@@ -43,18 +43,6 @@ public interface ISessionManager
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Reads mapped events for the central alarm monitor by attaching an internal
|
||||
/// (non-counted) distributor subscriber, so the alarm feed shares the one worker-event
|
||||
/// pump instead of opening a second raw drain of the single worker event channel.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Identifier of the session.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The mapped <see cref="MxEvent"/>s fanned by the session's distributor.</returns>
|
||||
IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Closes a session and terminates its worker process.</summary>
|
||||
/// <param name="sessionId">Identifier of the session to close.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -191,22 +190,6 @@ public sealed class SessionManager : ISessionManager
|
||||
return session.ReadEventsAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
|
||||
string sessionId,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
GatewaySession session = GetRequiredSession(sessionId);
|
||||
using IEventSubscriberLease lease = session.AttachInternalEventSubscriber();
|
||||
|
||||
await foreach (MxEvent mxEvent in lease.Reader
|
||||
.ReadAllAsync(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
yield return mxEvent;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<SessionCloseResult> CloseSessionAsync(
|
||||
string sessionId,
|
||||
|
||||
Reference in New Issue
Block a user