using System.Diagnostics.CodeAnalysis; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Alarms; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Grpc; using ZB.MOM.WW.MxGateway.Server.Metrics; using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Alarms; /// /// GWC-26 regression tests for the alarm monitor's startup ordering. Unlike the /// sibling alarm-monitor tests, the session manager here hands the monitor a REAL /// that is driven to Ready with a dashboard mirror, so the /// distributor pump is already running when the monitor attaches — the production /// condition under which a late internal subscriber silently misses everything the pump /// already fanned. /// public sealed class GatewayAlarmMonitorAttachOrderTests { private const string AlarmReference = "Galaxy!Area.Tank01.Hi"; private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(30); /// /// The monitor must take its internal distributor lease BEFORE issuing /// SubscribeAlarms, so transitions the worker emits during the /// subscribe + first-reconcile window buffer in the lease instead of being fanned to a /// subscriber set the monitor has not joined yet. The test parks the monitor inside its /// SubscribeAlarms round trip, emits a Raise and an Acknowledge, waits until the /// dashboard mirror proves the pump has already fanned both, and only then releases the /// monitor: with a late attach both transitions are lost to the alarm feed forever. /// /// A task that represents the asynchronous operation. [Fact] public async Task TransitionsDuringSubscribeWindow_StillReachTheAlarmFeed() { using GatewayMetrics metrics = new(); await using FakeSessionManager sessions = new(); sessions.HoldSubscribeUntilReleased(); using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics); using CancellationTokenSource cts = new(); await monitor.StartAsync(cts.Token); await sessions.WaitForSubscribeStartAsync(WaitTimeout); // A live feed subscriber, drained past its baseline ProviderStatus so it is registered // before any window transition is broadcast. List received = []; TaskCompletionSource baselineReceived = new(TaskCreationOptions.RunContinuationsAsynchronously); using CancellationTokenSource streamCts = new(); Task reader = ReadFeedAsync(monitor, received, baselineReceived, streamCts.Token); await baselineReceived.Task.WaitAsync(WaitTimeout); // The window: the worker reports a raise and an acknowledge while the monitor is still // waiting for its SubscribeAlarms reply. sessions.EmitEvent(Transition(1, AlarmTransitionKind.Raise)); sessions.EmitEvent(Transition(2, AlarmTransitionKind.Acknowledge)); // The dashboard mirror is an independent distributor subscriber: once it has both // events, the pump has provably already fanned them, so a subscriber that registers // after this point can never receive them. await WaitUntilAsync(() => sessions.Broadcaster.Captures.Count == 2, WaitTimeout); sessions.ReleaseSubscribe(); AlarmFeedMessage raise = await WaitForAsync( received, m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition && m.Transition.TransitionKind == AlarmTransitionKind.Raise, WaitTimeout); AlarmFeedMessage acknowledge = await WaitForAsync( received, m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition && m.Transition.TransitionKind == AlarmTransitionKind.Acknowledge, WaitTimeout); Assert.Equal(AlarmReference, raise.Transition.AlarmFullReference); Assert.Equal(AlarmReference, acknowledge.Transition.AlarmFullReference); await streamCts.CancelAsync(); await reader; await cts.CancelAsync(); await monitor.StopAsync(CancellationToken.None); } /// /// Defense in depth for any window the attach reorder cannot cover (worker restart, /// internal-subscriber overflow disconnect): when a reconcile snapshot reports an alarm /// the cache already holds but with the state advanced to /// , the monitor broadcasts an /// feed transition. Before the fix the /// acked state was absorbed silently by the snapshot replace, leaving live subscribers /// showing the alarm unacked until it cleared. /// /// A task that represents the asynchronous operation. [Fact] public async Task ApplyReconcileBroadcastsAcknowledgeDelta() { using GatewayMetrics metrics = new(); await using FakeSessionManager sessions = new(); using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics); using CancellationTokenSource cts = new(); await monitor.StartAsync(cts.Token); await sessions.WaitForSubscribeStartAsync(WaitTimeout); // Seed the cache with an active (unacked) alarm through a reconcile rather than a live // transition: a buffered live transition could still be in flight when the cache first // shows the alarm, and would then broadcast to the reader below and pollute the // exactly-one-transition assertion. A provider-mode event forces the reconcile // immediately, so the test never waits on the periodic timer. sessions.SetReconcileSnapshot(Snapshot(AlarmConditionState.Active)); sessions.EmitEvent(ProviderModeProbe(1)); await WaitUntilAsync( () => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference && alarm.CurrentState == AlarmConditionState.Active), WaitTimeout); // Subscribe AFTER the seed: this reader's snapshot carries the alarm, so every // transition it observes from here on is a reconcile-derived broadcast. List received = []; TaskCompletionSource snapshotComplete = new(TaskCreationOptions.RunContinuationsAsynchronously); using CancellationTokenSource streamCts = new(); Task reader = ReadFeedAsync(monitor, received, snapshotComplete, streamCts.Token, untilSnapshotComplete: true); await snapshotComplete.Task.WaitAsync(WaitTimeout); // The worker now reports the same alarm acknowledged. A provider-mode event forces an // immediate reconcile pass so the test does not wait on the periodic timer. sessions.SetReconcileSnapshot(Snapshot(AlarmConditionState.ActiveAcked)); sessions.EmitEvent(ProviderModeProbe(2)); AlarmFeedMessage acknowledge = await WaitForAsync( received, m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition && m.Transition.TransitionKind == AlarmTransitionKind.Acknowledge, WaitTimeout); Assert.Equal(AlarmReference, acknowledge.Transition.AlarmFullReference); lock (received) { AlarmFeedMessage[] transitions = received .Where(m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition) .ToArray(); AlarmFeedMessage single = Assert.Single(transitions); Assert.Equal(AlarmTransitionKind.Acknowledge, single.Transition.TransitionKind); } await streamCts.CancelAsync(); await reader; await cts.CancelAsync(); await monitor.StopAsync(CancellationToken.None); } /// /// NEXT-03. A reconcile Raise repair applied while the matching live Raise is still /// buffered must not double-broadcast: the live transition carrying the same worker /// timestamp and resulting state the cache already holds is a duplicate and is suppressed. /// /// A task that represents the asynchronous operation. [Fact] public async Task LiveTransitionMatchingReconcileRepair_IsSuppressed() { using GatewayMetrics metrics = new(); await using FakeSessionManager sessions = new(); using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics); using CancellationTokenSource cts = new(); await monitor.StartAsync(cts.Token); await sessions.WaitForSubscribeStartAsync(WaitTimeout); // The reconcile sees the raised alarm first (its Raise repair broadcasts before this // reader attaches) and stamps the cache with the worker's transition timestamp. Timestamp raiseTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 12, 0, 0, TimeSpan.Zero)); sessions.SetReconcileSnapshot(SnapshotAt(AlarmConditionState.Active, raiseTime, raiseTime)); sessions.EmitEvent(ProviderModeProbe(1)); await WaitUntilAsync( () => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference), WaitTimeout); List received = []; TaskCompletionSource snapshotComplete = new(TaskCreationOptions.RunContinuationsAsynchronously); using CancellationTokenSource streamCts = new(); Task reader = ReadFeedAsync(monitor, received, snapshotComplete, streamCts.Token, untilSnapshotComplete: true); await snapshotComplete.Task.WaitAsync(WaitTimeout); // The buffered live Raise drains with the same worker timestamp — a duplicate of the // repair. The follow-up Acknowledge with a newer timestamp is genuine and must pass. sessions.EmitEvent(TransitionAt(2, AlarmTransitionKind.Raise, raiseTime, raiseTime)); Timestamp ackTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 12, 0, 5, TimeSpan.Zero)); sessions.EmitEvent(TransitionAt(3, AlarmTransitionKind.Acknowledge, ackTime, raiseTime)); await WaitForAsync( received, m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition && m.Transition.TransitionKind == AlarmTransitionKind.Acknowledge, WaitTimeout); lock (received) { AlarmFeedMessage[] transitions = received .Where(m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition) .ToArray(); AlarmFeedMessage single = Assert.Single(transitions); Assert.Equal(AlarmTransitionKind.Acknowledge, single.Transition.TransitionKind); } await streamCts.CancelAsync(); await reader; await cts.CancelAsync(); await monitor.StopAsync(CancellationToken.None); } /// /// NEXT-03, the Clear leg. A reconcile Clear repair removes the cache entry, so the /// buffered live Clear is deduped through the tombstone keyed on the instance's original /// raise timestamp — and a genuinely new raise/clear cycle is never swallowed. /// /// A task that represents the asynchronous operation. [Fact] public async Task LiveClearMatchingReconcileClearRepair_IsSuppressed() { using GatewayMetrics metrics = new(); await using FakeSessionManager sessions = new(); using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics); using CancellationTokenSource cts = new(); await monitor.StartAsync(cts.Token); await sessions.WaitForSubscribeStartAsync(WaitTimeout); Timestamp raiseTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 13, 0, 0, TimeSpan.Zero)); sessions.SetReconcileSnapshot(SnapshotAt(AlarmConditionState.Active, raiseTime, raiseTime)); sessions.EmitEvent(ProviderModeProbe(1)); await WaitUntilAsync( () => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference), WaitTimeout); List received = []; TaskCompletionSource snapshotComplete = new(TaskCreationOptions.RunContinuationsAsynchronously); using CancellationTokenSource streamCts = new(); Task reader = ReadFeedAsync(monitor, received, snapshotComplete, streamCts.Token, untilSnapshotComplete: true); await snapshotComplete.Task.WaitAsync(WaitTimeout); // The worker no longer reports the alarm: the reconcile synthesizes the Clear repair and // tombstones the instance by its original raise timestamp. sessions.SetReconcileSnapshot(); sessions.EmitEvent(ProviderModeProbe(2)); await WaitForAsync( received, m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition && m.Transition.TransitionKind == AlarmTransitionKind.Clear, WaitTimeout); // The buffered live Clear for the SAME instance is a duplicate of the repair; the Raise // that follows starts a new instance and must pass. Timestamp clearTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 13, 0, 10, TimeSpan.Zero)); sessions.EmitEvent(TransitionAt(3, AlarmTransitionKind.Clear, clearTime, raiseTime)); Timestamp newRaiseTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 13, 0, 20, TimeSpan.Zero)); sessions.EmitEvent(TransitionAt(4, AlarmTransitionKind.Raise, newRaiseTime, newRaiseTime)); await WaitForAsync( received, m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition && m.Transition.TransitionKind == AlarmTransitionKind.Raise, WaitTimeout); lock (received) { AlarmTransitionKind[] kinds = received .Where(m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition) .Select(m => m.Transition.TransitionKind) .ToArray(); Assert.Equal([AlarmTransitionKind.Clear, AlarmTransitionKind.Raise], kinds); } await streamCts.CancelAsync(); await reader; await cts.CancelAsync(); await monitor.StopAsync(CancellationToken.None); } private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics) { AlarmsOptions options = new() { Enabled = true, SubscriptionExpression = @"\\NODE\Galaxy!Area", }; return new GatewayAlarmMonitor( sessions, new StubWatchListResolver(), metrics, Microsoft.Extensions.Options.Options.Create(new GatewayOptions { Alarms = options }), NullLogger.Instance); } // Drains the monitor's feed into received, signalling gate on the first message (the // baseline ProviderStatus) or, when untilSnapshotComplete is set, on SnapshotComplete. private static Task ReadFeedAsync( GatewayAlarmMonitor monitor, List received, TaskCompletionSource gate, CancellationToken cancellationToken, bool untilSnapshotComplete = false) { return Task.Run( async () => { try { await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, cancellationToken)) { bool opensGate = !untilSnapshotComplete || message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.SnapshotComplete; // Record only what arrives AFTER the gate opened: everything up to and // including the gate message is this subscriber's snapshot preamble, not // a live broadcast, so assertions stay about broadcasts alone. lock (received) { if (gate.Task.IsCompleted) { received.Add(message); } } if (opensGate) { gate.TrySetResult(); } } } catch (OperationCanceledException) { // Expected when the test cancels the stream. } }, CancellationToken.None); } private static MxEvent Transition(ulong sequence, AlarmTransitionKind kind) => new() { Family = MxEventFamily.OnAlarmTransition, WorkerSequence = sequence, OnAlarmTransition = new OnAlarmTransitionEvent { AlarmFullReference = AlarmReference, SourceObjectReference = "Tank01", AlarmTypeName = "AnalogLimitAlarm.Hi", TransitionKind = kind, Severity = 500, SourceProvider = AlarmProviderMode.Alarmmgr, TransitionTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), }, }; // A no-op provider-mode event. The monitor forces an immediate reconcile after every one, // which is how these tests drive a reconcile pass without waiting on the periodic timer. private static MxEvent ProviderModeProbe(ulong sequence) => new() { Family = MxEventFamily.OnAlarmProviderModeChanged, WorkerSequence = sequence, OnAlarmProviderModeChanged = new OnAlarmProviderModeChangedEvent { Mode = AlarmProviderMode.Alarmmgr, Reason = "probe", At = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), }, }; private static ActiveAlarmSnapshot Snapshot(AlarmConditionState state) => new() { AlarmFullReference = AlarmReference, SourceObjectReference = "Tank01", AlarmTypeName = "AnalogLimitAlarm.Hi", CurrentState = state, Severity = 500, SourceProvider = AlarmProviderMode.Alarmmgr, }; // Snapshot carrying the worker-side identity markers the NEXT-03 dedup compares on. private static ActiveAlarmSnapshot SnapshotAt( AlarmConditionState state, Timestamp lastTransition, Timestamp originalRaise) { ActiveAlarmSnapshot snapshot = Snapshot(state); snapshot.LastTransitionTimestamp = lastTransition; snapshot.OriginalRaiseTimestamp = originalRaise; return snapshot; } // Live transition with explicit worker timestamps, for driving the NEXT-03 dedup. private static MxEvent TransitionAt( ulong sequence, AlarmTransitionKind kind, Timestamp transitionTimestamp, Timestamp originalRaise) { MxEvent mxEvent = Transition(sequence, kind); mxEvent.OnAlarmTransition.TransitionTimestamp = transitionTimestamp; mxEvent.OnAlarmTransition.OriginalRaiseTimestamp = originalRaise; return mxEvent; } private static async Task WaitForAsync( List received, Func predicate, TimeSpan timeout) { DateTime deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { lock (received) { AlarmFeedMessage? match = received.FirstOrDefault(predicate); if (match is not null) { return match; } } await Task.Delay(25); } throw new TimeoutException("No matching AlarmFeedMessage was received in time."); } private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) { DateTime deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (condition()) { return; } await Task.Delay(25); } throw new TimeoutException("Condition was not met in time."); } /// that resolves an empty watch-list. private sealed class StubWatchListResolver : IAlarmWatchListResolver { /// public Task> ResolveAsync( AlarmsOptions options, CancellationToken cancellationToken = default) => Task.FromResult>([]); } /// /// Session manager that hands the monitor a real driven to /// Ready with a dashboard mirror, so the distributor pump is running before the monitor /// attaches. pushes worker events through the fake worker client /// into that pump, exactly as a live worker would. /// private sealed class FakeSessionManager : ISessionManager, IAsyncDisposable { private readonly Channel _events = Channel.CreateUnbounded(); private readonly TaskCompletionSource _subscribeStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly object _sync = new(); private TaskCompletionSource _subscribeGate = CreateReleasedGate(); private ActiveAlarmSnapshot[] _reconcileSnapshot = []; private GatewaySession? _session; /// Dashboard mirror attached to the session; proves what the pump has fanned. public RecordingDashboardEventBroadcaster Broadcaster { get; } = new(); /// Re-arms the gate so SubscribeAlarms parks until . public void HoldSubscribeUntilReleased() => _subscribeGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); /// Releases a gate armed by . public void ReleaseSubscribe() => _subscribeGate.TrySetResult(); /// Completes once the monitor's SubscribeAlarms command has arrived. /// The maximum time to wait. /// A task that completes when the command arrives. public Task WaitForSubscribeStartAsync(TimeSpan timeout) => _subscribeStarted.Task.WaitAsync(timeout); /// Sets the active-alarm snapshot every QueryActiveAlarms reconcile returns. /// The snapshots to report. public void SetReconcileSnapshot(params ActiveAlarmSnapshot[] snapshots) { lock (_sync) { _reconcileSnapshot = snapshots; } } /// Pushes a worker event into the session's distributor pump. /// The event to push. public void EmitEvent(MxEvent mxEvent) => _events.Writer.TryWrite(new WorkerEvent { Event = mxEvent }); /// public Task OpenSessionAsync( SessionOpenRequest request, string? clientIdentity, string? ownerKeyId, CancellationToken cancellationToken) { GatewaySession session = new( sessionId: "session-alarm-attach-order", backendName: "Galaxy", pipeName: "mxaccess-gateway-1-session-alarm-attach-order", nonce: "nonce", clientIdentity: clientIdentity, ownerKeyId: ownerKeyId, clientSessionName: request.ClientSessionName, clientCorrelationId: request.ClientCorrelationId, commandTimeout: TimeSpan.FromSeconds(30), startupTimeout: TimeSpan.FromSeconds(30), shutdownTimeout: TimeSpan.FromSeconds(30), leaseDuration: TimeSpan.FromMinutes(30), openedAt: DateTimeOffset.UtcNow, eventStreaming: new SessionEventStreaming( new MxAccessGrpcMapper(), new EventOptions { QueueCapacity = 64 }, NullLogger.Instance, TimeProvider.System, new GatewayMetrics(), Broadcaster)); session.AttachWorkerClient(new ChannelWorkerClient(session.SessionId, _events.Reader)); // MarkReady starts the dashboard mirror, and with it the distributor pump — the // production precondition this regression depends on. session.MarkReady(); _session = session; return Task.FromResult(session); } /// public async Task InvokeAsync( string sessionId, WorkerCommand command, CancellationToken cancellationToken) { MxCommandReply reply = new() { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, }; switch (command.Command?.Kind) { case MxCommandKind.SubscribeAlarms: _subscribeStarted.TrySetResult(); await _subscribeGate.Task.WaitAsync(cancellationToken).ConfigureAwait(false); break; case MxCommandKind.QueryActiveAlarms: QueryActiveAlarmsReplyPayload payload = new(); lock (_sync) { payload.Snapshots.AddRange(_reconcileSnapshot.Select(snapshot => snapshot.Clone())); } reply.QueryActiveAlarms = payload; break; } return new WorkerCommandReply { Reply = reply }; } /// public IAsyncEnumerable ReadEventsAsync( string sessionId, CancellationToken cancellationToken) => throw new NotSupportedException(); /// public bool TryGetSession(string sessionId, [MaybeNullWhen(false)] out GatewaySession session) { session = _session; return session is not null; } /// public Task CloseSessionAsync(string sessionId, CancellationToken cancellationToken) { _events.Writer.TryComplete(); return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); } /// public Task KillWorkerAsync(string sessionId, string reason, CancellationToken cancellationToken) => Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); /// public Task CloseExpiredLeasesAsync(DateTimeOffset now, CancellationToken cancellationToken) => Task.FromResult(0); /// public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask; /// Disposes the session the fake handed out. /// A task that represents the asynchronous operation. public async ValueTask DisposeAsync() { _events.Writer.TryComplete(); if (_session is not null) { await _session.DisposeAsync().ConfigureAwait(false); } } private static TaskCompletionSource CreateReleasedGate() { TaskCompletionSource gate = new(TaskCreationOptions.RunContinuationsAsynchronously); gate.SetResult(); return gate; } } }