fix(alarms): dedup reconcile/live duplicate broadcasts on the alarm feed (NEXT-03)
A periodic reconcile can synthesize a repair transition whose matching live transition is still buffered in the alarm lease; both then broadcast as indistinguishable duplicates on StreamAlarms and the dashboard hub. Nothing serializes the two paths, and a correct serialization needs a worker-side high-water mark on QueryActiveAlarms (proto + worker change + a stall path), so this closes the common case with a local best-effort dedup instead: both paths already carry the same worker-derived identity — the worker stamps record.TransitionTimestampUtc into both OnAlarmTransitionEvent's transition_timestamp and ActiveAlarmSnapshot.last_transition_timestamp — so ApplyTransition suppresses a live transition whose (timestamp, resulting state) the cache already carries from a repair. The Clear leg has no cache entry left to compare, so ApplyReconcile tombstones each synthesized Clear by the instance's original_raise_timestamp for one reconcile generation; a matching live Clear consumes the tombstone, while a new raise/clear cycle carries a newer raise timestamp and passes. Suppression fires only on a positive marker match — unset timestamps keep today's behavior — so the documented at-least-once consumer contract stands (gateway.md, Sessions.md updated in the same change). Tests: two new regressions drive the exact race through the GWC-26 harness (repair-then-buffered-live for Raise and for Clear, each with a genuine follow-up transition proving no over-suppression); GatewayAlarmMonitor suites 18/18.
This commit is contained in:
Joseph Doherty
@@ -155,6 +155,133 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
|
||||
await monitor.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// NEXT-03. A reconcile Raise repair applied while the matching live Raise is still
|
||||
/// buffered must not double-broadcast: the live transition carrying the same worker
|
||||
/// timestamp and resulting state the cache already holds is a duplicate and is suppressed.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LiveTransitionMatchingReconcileRepair_IsSuppressed()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
await using FakeSessionManager sessions = new();
|
||||
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await monitor.StartAsync(cts.Token);
|
||||
await sessions.WaitForSubscribeStartAsync(WaitTimeout);
|
||||
|
||||
// The reconcile sees the raised alarm first (its Raise repair broadcasts before this
|
||||
// reader attaches) and stamps the cache with the worker's transition timestamp.
|
||||
Timestamp raiseTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 12, 0, 0, TimeSpan.Zero));
|
||||
sessions.SetReconcileSnapshot(SnapshotAt(AlarmConditionState.Active, raiseTime, raiseTime));
|
||||
sessions.EmitEvent(ProviderModeProbe(1));
|
||||
await WaitUntilAsync(
|
||||
() => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference),
|
||||
WaitTimeout);
|
||||
|
||||
List<AlarmFeedMessage> received = [];
|
||||
TaskCompletionSource snapshotComplete = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using CancellationTokenSource streamCts = new();
|
||||
Task reader = ReadFeedAsync(monitor, received, snapshotComplete, streamCts.Token, untilSnapshotComplete: true);
|
||||
await snapshotComplete.Task.WaitAsync(WaitTimeout);
|
||||
|
||||
// The buffered live Raise drains with the same worker timestamp — a duplicate of the
|
||||
// repair. The follow-up Acknowledge with a newer timestamp is genuine and must pass.
|
||||
sessions.EmitEvent(TransitionAt(2, AlarmTransitionKind.Raise, raiseTime, raiseTime));
|
||||
Timestamp ackTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 12, 0, 5, TimeSpan.Zero));
|
||||
sessions.EmitEvent(TransitionAt(3, AlarmTransitionKind.Acknowledge, ackTime, raiseTime));
|
||||
|
||||
await WaitForAsync(
|
||||
received,
|
||||
m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition
|
||||
&& m.Transition.TransitionKind == AlarmTransitionKind.Acknowledge,
|
||||
WaitTimeout);
|
||||
|
||||
lock (received)
|
||||
{
|
||||
AlarmFeedMessage[] transitions = received
|
||||
.Where(m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition)
|
||||
.ToArray();
|
||||
AlarmFeedMessage single = Assert.Single(transitions);
|
||||
Assert.Equal(AlarmTransitionKind.Acknowledge, single.Transition.TransitionKind);
|
||||
}
|
||||
|
||||
await streamCts.CancelAsync();
|
||||
await reader;
|
||||
await cts.CancelAsync();
|
||||
await monitor.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// NEXT-03, the Clear leg. A reconcile Clear repair removes the cache entry, so the
|
||||
/// buffered live Clear is deduped through the tombstone keyed on the instance's original
|
||||
/// raise timestamp — and a genuinely new raise/clear cycle is never swallowed.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LiveClearMatchingReconcileClearRepair_IsSuppressed()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
await using FakeSessionManager sessions = new();
|
||||
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await monitor.StartAsync(cts.Token);
|
||||
await sessions.WaitForSubscribeStartAsync(WaitTimeout);
|
||||
|
||||
Timestamp raiseTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 13, 0, 0, TimeSpan.Zero));
|
||||
sessions.SetReconcileSnapshot(SnapshotAt(AlarmConditionState.Active, raiseTime, raiseTime));
|
||||
sessions.EmitEvent(ProviderModeProbe(1));
|
||||
await WaitUntilAsync(
|
||||
() => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference),
|
||||
WaitTimeout);
|
||||
|
||||
List<AlarmFeedMessage> received = [];
|
||||
TaskCompletionSource snapshotComplete = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using CancellationTokenSource streamCts = new();
|
||||
Task reader = ReadFeedAsync(monitor, received, snapshotComplete, streamCts.Token, untilSnapshotComplete: true);
|
||||
await snapshotComplete.Task.WaitAsync(WaitTimeout);
|
||||
|
||||
// The worker no longer reports the alarm: the reconcile synthesizes the Clear repair and
|
||||
// tombstones the instance by its original raise timestamp.
|
||||
sessions.SetReconcileSnapshot();
|
||||
sessions.EmitEvent(ProviderModeProbe(2));
|
||||
await WaitForAsync(
|
||||
received,
|
||||
m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition
|
||||
&& m.Transition.TransitionKind == AlarmTransitionKind.Clear,
|
||||
WaitTimeout);
|
||||
|
||||
// The buffered live Clear for the SAME instance is a duplicate of the repair; the Raise
|
||||
// that follows starts a new instance and must pass.
|
||||
Timestamp clearTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 13, 0, 10, TimeSpan.Zero));
|
||||
sessions.EmitEvent(TransitionAt(3, AlarmTransitionKind.Clear, clearTime, raiseTime));
|
||||
Timestamp newRaiseTime = Timestamp.FromDateTimeOffset(new DateTimeOffset(2026, 8, 10, 13, 0, 20, TimeSpan.Zero));
|
||||
sessions.EmitEvent(TransitionAt(4, AlarmTransitionKind.Raise, newRaiseTime, newRaiseTime));
|
||||
|
||||
await WaitForAsync(
|
||||
received,
|
||||
m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition
|
||||
&& m.Transition.TransitionKind == AlarmTransitionKind.Raise,
|
||||
WaitTimeout);
|
||||
|
||||
lock (received)
|
||||
{
|
||||
AlarmTransitionKind[] kinds = received
|
||||
.Where(m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.Transition)
|
||||
.Select(m => m.Transition.TransitionKind)
|
||||
.ToArray();
|
||||
Assert.Equal([AlarmTransitionKind.Clear, AlarmTransitionKind.Raise], kinds);
|
||||
}
|
||||
|
||||
await streamCts.CancelAsync();
|
||||
await reader;
|
||||
await cts.CancelAsync();
|
||||
await monitor.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics)
|
||||
{
|
||||
AlarmsOptions options = new()
|
||||
@@ -254,6 +381,31 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
|
||||
SourceProvider = AlarmProviderMode.Alarmmgr,
|
||||
};
|
||||
|
||||
// Snapshot carrying the worker-side identity markers the NEXT-03 dedup compares on.
|
||||
private static ActiveAlarmSnapshot SnapshotAt(
|
||||
AlarmConditionState state,
|
||||
Timestamp lastTransition,
|
||||
Timestamp originalRaise)
|
||||
{
|
||||
ActiveAlarmSnapshot snapshot = Snapshot(state);
|
||||
snapshot.LastTransitionTimestamp = lastTransition;
|
||||
snapshot.OriginalRaiseTimestamp = originalRaise;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
// Live transition with explicit worker timestamps, for driving the NEXT-03 dedup.
|
||||
private static MxEvent TransitionAt(
|
||||
ulong sequence,
|
||||
AlarmTransitionKind kind,
|
||||
Timestamp transitionTimestamp,
|
||||
Timestamp originalRaise)
|
||||
{
|
||||
MxEvent mxEvent = Transition(sequence, kind);
|
||||
mxEvent.OnAlarmTransition.TransitionTimestamp = transitionTimestamp;
|
||||
mxEvent.OnAlarmTransition.OriginalRaiseTimestamp = originalRaise;
|
||||
return mxEvent;
|
||||
}
|
||||
|
||||
private static async Task<AlarmFeedMessage> WaitForAsync(
|
||||
List<AlarmFeedMessage> received,
|
||||
Func<AlarmFeedMessage, bool> predicate,
|
||||
|
||||
Reference in New Issue
Block a user