dohertj2/mxaccessgw
1
0
Fork 0
Code Issues 14 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(GWC-26): attach the alarm monitor's lease before SubscribeAlarms

Browse Source 
RunMonitorAsync issued SubscribeAlarms and the first reconcile before the
internal distributor subscriber was attached (via ISessionManager
.ReadAlarmEventsAsync). The pump has been running since MarkReady started the
dashboard mirror and only fans to subscribers registered at fan-out time, so
every transition raised in that two-round-trip window bypassed the alarm feed —
and a missed Acknowledge was never repaired, because ApplyReconcile broadcast
presence deltas only.

- The monitor now takes the internal lease directly from its session BEFORE
  SubscribeAlarms and drains it after the first reconcile; window transitions
  buffer in the lease's bounded channel. Processing them after ApplyReconcile is
  order-safe (ApplyTransition handles alarms the snapshot already placed).
- ISessionManager.ReadAlarmEventsAsync removed — zero remaining callers.
- ApplyReconcile broadcasts an Acknowledge feed transition when a both-present
  alarm's state advanced to ActiveAcked. This is a feed-level repair on the
  AlarmFeedMessage/StreamAlarms surface rebuilt from the worker's own snapshot,
  not MxEvent emission, so the "never synthesize events" rule is untouched;
  the reasoning is recorded on ApplyReconcile.

The alarm-monitor test fakes now hand the monitor a real Ready GatewaySession
with a dashboard mirror, which is what makes the window reproducible.

Docs: docs/Sessions.md and gateway.md alarm-monitor ordering notes.

Refs: archreview/2026-07-12/remediation/10-gateway-core.md GWC-26
This commit is contained in:
Joseph Doherty
2026-08-07 05:40:33 -04:00
parent 1a63fdd7db
commit 3b6a239ed6
16 changed files with 674 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs
+34 -4
View File
@@ -210,6 +210,17 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
try
{
// Attach the internal distributor subscriber BEFORE subscribing (GWC-26). The pump
// has been running since MarkReady started the dashboard mirror, and the distributor
// only fans to subscribers registered at the time of the fan-out, so a subscriber
// taken after SubscribeAlarms + the first reconcile would silently lose every
// transition raised inside that two-round-trip window — and a missed Acknowledge is
// never repaired by the presence-only reconcile deltas. Transitions arriving while we
// subscribe and reconcile simply buffer in this lease's bounded channel; if it ever
// overflowed, the internal subscriber is disconnected (it never faults the session),
// the enumeration below ends, and the supervisor loop restarts the lifecycle.
using IEventSubscriberLease alarmLease = session.AttachInternalEventSubscriber();
await SubscribeAlarmsAsync(session.SessionId, subscription, stoppingToken).ConfigureAwait(false);
await ReconcileAsync(session.SessionId, stoppingToken).ConfigureAwait(false);
@@ -228,9 +239,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
// Consume mapped MxEvents through the session's single distributor pump (as an
// internal, non-counted subscriber) rather than opening a second raw drain of the
// worker event channel — a second drain would split events with the dashboard
// mirror pump and silently lose Acknowledge/mode-change transitions.
await foreach (MxEvent mxEvent in _sessionManager
.ReadAlarmEventsAsync(session.SessionId, linked.Token)
// mirror pump and silently lose Acknowledge/mode-change transitions. The lease was
// taken above, before SubscribeAlarms; draining it only now is order-safe because
// ApplyTransition handles alarms the reconcile already placed in the cache.
await foreach (MxEvent mxEvent in alarmLease.Reader
.ReadAllAsync(linked.Token)
.ConfigureAwait(false))
{
if (mxEvent is { BodyCase: MxEvent.BodyOneofCase.OnAlarmTransition }
@@ -511,6 +524,12 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
// Replaces the cache with the worker's authoritative snapshot, broadcasting
// a synthetic transition for any alarm the live stream missed.
//
// Repair scope (GWC-26): presence deltas (Clear/Raise) plus acked-state deltas. These are
// ALARM FEED transitions (AlarmFeedMessage on the StreamAlarms/dashboard surface), rebuilt
// from the worker's own authoritative snapshot to repair what the live feed missed. They are
// not MxEvents and never reach the gRPC StreamEvents path, so this feed-level repair does not
// breach the "never synthesize events" rule, which governs MxEvent emission.
private void ApplyReconcile(IEnumerable<ActiveAlarmSnapshot> snapshots)
{
Dictionary<string, ActiveAlarmSnapshot> next = new(StringComparer.Ordinal);
@@ -536,12 +555,23 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
foreach (KeyValuePair<string, ActiveAlarmSnapshot> incoming in next)
{
if (!_alarms.ContainsKey(incoming.Key))
if (!_alarms.TryGetValue(incoming.Key, out ActiveAlarmSnapshot? existing))
{
Broadcast(
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Raise) },
incoming.Key);
}
else if (existing.CurrentState != incoming.Value.CurrentState
&& incoming.Value.CurrentState == AlarmConditionState.ActiveAcked)
{
// The alarm was already known but the worker now reports it acknowledged: the
// live Acknowledge transition never reached the feed. Without this the acked
// state is absorbed silently by the snapshot replace below and subscribers show
// the alarm unacked until it clears.
Broadcast(
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Acknowledge) },
incoming.Key);
}
}
_alarms.Clear();
src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs
-12
View File
@@ -43,18 +43,6 @@ public interface ISessionManager
string sessionId,
CancellationToken cancellationToken);
/// <summary>
/// Reads mapped events for the central alarm monitor by attaching an internal
/// (non-counted) distributor subscriber, so the alarm feed shares the one worker-event
/// pump instead of opening a second raw drain of the single worker event channel.
/// </summary>
/// <param name="sessionId">Identifier of the session.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>The mapped <see cref="MxEvent"/>s fanned by the session's distributor.</returns>
IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
string sessionId,
CancellationToken cancellationToken);
/// <summary>Closes a session and terminates its worker process.</summary>
/// <param name="sessionId">Identifier of the session to close.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs
-17
View File
@@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging;
@@ -191,22 +190,6 @@ public sealed class SessionManager : ISessionManager
return session.ReadEventsAsync(cancellationToken);
}
/// <inheritdoc />
public async IAsyncEnumerable<MxEvent> ReadAlarmEventsAsync(
string sessionId,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
GatewaySession session = GetRequiredSession(sessionId);
using IEventSubscriberLease lease = session.AttachInternalEventSubscriber();
await foreach (MxEvent mxEvent in lease.Reader
.ReadAllAsync(cancellationToken)
.ConfigureAwait(false))
{
yield return mxEvent;
}
}
/// <inheritdoc />
public async Task<SessionCloseResult> CloseSessionAsync(
string sessionId,