fix(GWC-35): repair re-raised alarm snapshots

This commit is contained in:
Joseph Doherty
2026-08-25 10:58:21 -04:00
parent b0156c685e
commit 780720165a
2 changed files with 69 additions and 7 deletions
@@ -839,15 +839,17 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Raise) },
incoming.Key);
}
else if (existing.CurrentState != incoming.Value.CurrentState
&& incoming.Value.CurrentState == AlarmConditionState.ActiveAcked)
else if (existing.CurrentState != incoming.Value.CurrentState)
{
// The alarm was already known but the worker now reports it acknowledged: the
// live Acknowledge transition never reached the feed. Without this the acked
// state is absorbed silently by the snapshot replace below and subscribers show
// the alarm unacked until it clears.
// The alarm was already known but the worker now reports a different state. An
// ActiveAcked snapshot repairs a missed Acknowledge. Any other incoming active
// state repairs a missed Raise for the incoming instance (including an
// acknowledged alarm reference that was cleared and re-raised between passes).
AlarmTransitionKind kind = incoming.Value.CurrentState == AlarmConditionState.ActiveAcked
? AlarmTransitionKind.Acknowledge
: AlarmTransitionKind.Raise;
Broadcast(
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Acknowledge) },
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, kind) },
incoming.Key);
}
}
@@ -155,6 +155,66 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// GWC-35: an acknowledged cached alarm followed by an active snapshot with a newer
/// original-raise timestamp is a new alarm instance, not an acknowledgement reversal. The
/// reconcile repair must broadcast a Raise carrying the incoming instance identity.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ApplyReconcile_WhenAcknowledgedAlarmHasNewActiveInstance_BroadcastsRaise()
{
using GatewayMetrics metrics = new();
await using FakeSessionManager sessions = new();
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForSubscribeStartAsync(WaitTimeout);
Timestamp oldRaise = Timestamp.FromDateTimeOffset(
new DateTimeOffset(2026, 8, 10, 13, 0, 0, TimeSpan.Zero));
sessions.SetReconcileSnapshot(SnapshotAt(AlarmConditionState.ActiveAcked, oldRaise, oldRaise));
sessions.EmitEvent(ProviderModeProbe(1));
await WaitUntilAsync(
() => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference
&& alarm.CurrentState == AlarmConditionState.ActiveAcked),
WaitTimeout);
List<AlarmFeedMessage> received = [];
TaskCompletionSource snapshotComplete = new(TaskCreationOptions.RunContinuationsAsynchronously);
using CancellationTokenSource streamCts = new();
Task reader = ReadFeedAsync(monitor, received, snapshotComplete, streamCts.Token, untilSnapshotComplete: true);
await snapshotComplete.Task.WaitAsync(WaitTimeout);
Timestamp newRaise = Timestamp.FromDateTimeOffset(
new DateTimeOffset(2026, 8, 10, 13, 0, 20, TimeSpan.Zero));
sessions.SetReconcileSnapshot(SnapshotAt(AlarmConditionState.Active, newRaise, newRaise));
sessions.EmitEvent(ProviderModeProbe(2));
AlarmFeedMessage raise = await WaitForAsync(
received,
m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition
&& m.Transition.TransitionKind == AlarmTransitionKind.Raise,
WaitTimeout);
Assert.Equal(AlarmReference, raise.Transition.AlarmFullReference);
Assert.Equal(newRaise, raise.Transition.OriginalRaiseTimestamp);
Assert.Equal(newRaise, raise.Transition.TransitionTimestamp);
lock (received)
{
AlarmFeedMessage[] transitions = received
.Where(m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition)
.ToArray();
Assert.Single(transitions);
}
await streamCts.CancelAsync();
await reader;
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// NEXT-03. A reconcile Raise repair applied while the matching live Raise is still
/// buffered must not double-broadcast: the live transition carrying the same worker