feat(GWC-37,WRK-32): refresh live alarm watch lists

src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs

@@ -646,6 +646,7 @@ public sealed class GatewayAlarmMonitorProviderModeTests
 
         // The empty list still went to the worker — the verdict is a report, not a behaviour change.
         Assert.Empty(sessions.LastSubscribeCommand!.WatchList);
+        Assert.True(sessions.LastSubscribeCommand.FallbackEnabled);
 
         AlarmProviderStatus status = await ReadFirstProviderStatusAsync(monitor);
 
@@ -661,16 +662,14 @@ public sealed class GatewayAlarmMonitorProviderModeTests
 
     /// <summary>
     ///     GWC-37: the failover edge is the moment the watch-list stops being theoretical, so the
-    ///     monitor re-resolves it there and publishes the fresh verdict on the same status frame
-    ///     the failover produces. Discovery that has recovered since <c>SubscribeAlarms</c> is
-    ///     reported as drift rather than applied — the worker rejects a second <c>SubscribeAlarms</c>
-    ///     while subscribed, and the only way to replace the list rebuilds the failover consumer on
-    ///     the primary that just failed. The status names both the fresh count and the sent count so
-    ///     an operator knows a monitor restart will pick the new list up.
+    ///     monitor re-resolves it there, hot-swaps the recovered list through
+    ///     <c>UpdateAlarmWatchList</c>, and publishes the acknowledged verdict on the same status
+    ///     frame the failover produces. The update must precede the status frame so the frame never
+    ///     claims recovered coverage before the worker has accepted it.
     /// </summary>
     /// <returns>A task that represents the asynchronous operation.</returns>
     [Fact]
-    public async Task SubtagCoverage_WhenDiscoveryRecoversByFailover_ReResolvesAndReportsDrift()
+    public async Task SubtagCoverage_WhenDiscoveryRecoversByFailover_UpdatesWorkerBeforePublishingStatus()
     {
         using GatewayMetrics metrics = new();
         FakeSessionManager sessions = new();
@@ -734,13 +733,11 @@ public sealed class GatewayAlarmMonitorProviderModeTests
         // The re-resolve happened on the failover edge, before the status was published.
         Assert.Equal(2, resolver.Calls);
 
+        Assert.Single(sessions.UpdateCommands);
+        Assert.Equal(2, sessions.UpdateCommands[0].WatchList.Count);
         string reason = degraded.ProviderStatus.Reason;
         Assert.Contains("alarmmgr failed", reason, StringComparison.Ordinal);
-        Assert.Contains(AlarmProviderReasons.SubtagWatchListStale, reason, StringComparison.Ordinal);
-        Assert.Contains("2 target(s) now resolve against 0 sent", reason, StringComparison.Ordinal);
-
-        // The stale-list warning replaced the empty/failed pair: discovery works again, so the only
-        // remaining problem is that the worker is still using the list it was handed.
+        Assert.DoesNotContain(AlarmProviderReasons.SubtagWatchListStale, reason, StringComparison.Ordinal);
         Assert.DoesNotContain(AlarmProviderReasons.SubtagWatchListEmpty, reason, StringComparison.Ordinal);
         Assert.DoesNotContain(AlarmProviderReasons.SubtagDiscoveryFailed, reason, StringComparison.Ordinal);
 
@@ -750,6 +747,69 @@ public sealed class GatewayAlarmMonitorProviderModeTests
         await monitor.StopAsync(CancellationToken.None);
     }
 
+    /// <summary>Periodic reconciliation applies a recovered watch list once and suppresses an unchanged resolution on the next cadence.</summary>
+    [Fact]
+    public async Task WatchListRefresh_PeriodicRecovery_SendsExactlyOneMaterialUpdate()
+    {
+        using GatewayMetrics metrics = new();
+        FakeSessionManager sessions = new();
+        AlarmWatchListResolution recovered = new(
+            [Target("Galaxy!Area.Tank01.Level.HiHi")],
+            DiscoveryAttempted: true,
+            DiscoveryFailed: false,
+            DiscoveryFailureMessage: null);
+        ScriptedWatchListResolver resolver = new(DiscoveryFailedResolution(), recovered, recovered);
+        using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver, reconcileIntervalSeconds: 5);
+
+        using CancellationTokenSource cts = new();
+        await monitor.StartAsync(cts.Token);
+        await sessions.WaitForUpdateCountAsync(1, WaitTimeout);
+        await WaitUntilAsync(() => resolver.Calls >= 3, WaitTimeout);
+
+        Assert.Single(sessions.UpdateCommands);
+        Assert.Single(sessions.UpdateCommands[0].WatchList);
+
+        await cts.CancelAsync();
+        await monitor.StopAsync(CancellationToken.None);
+    }
+
+    /// <summary>A rejected refresh keeps the old fingerprint and degraded verdict so the same material change is retried.</summary>
+    [Fact]
+    public async Task WatchListRefresh_WorkerRejects_RetainsFingerprintAndRetriesSameTargets()
+    {
+        using GatewayMetrics metrics = new();
+        FakeSessionManager sessions = new();
+        sessions.UpdateReplyCodes.Enqueue(ProtocolStatusCode.MxaccessFailure);
+        sessions.UpdateReplyCodes.Enqueue(ProtocolStatusCode.Ok);
+        AlarmWatchListResolution recovered = new(
+            [Target("Galaxy!Area.Tank01.Level.HiHi")],
+            DiscoveryAttempted: true,
+            DiscoveryFailed: false,
+            DiscoveryFailureMessage: null);
+        ScriptedWatchListResolver resolver = new(DiscoveryFailedResolution(), recovered, recovered);
+        using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver);
+
+        using CancellationTokenSource cts = new();
+        await monitor.StartAsync(cts.Token);
+        await sessions.WaitForSubscribeAsync(WaitTimeout);
+
+        sessions.EmitModeChange(AlarmProviderMode.Subtag, "alarmmgr failed");
+        await sessions.WaitForUpdateCountAsync(1, WaitTimeout);
+        AlarmProviderStatus rejected = await ReadFirstProviderStatusAsync(monitor);
+        Assert.True(rejected.Degraded);
+        Assert.Contains(AlarmProviderReasons.SubtagWatchListStale, rejected.Reason, StringComparison.Ordinal);
+
+        sessions.EmitModeChange(AlarmProviderMode.Subtag, "alarmmgr still failed");
+        await sessions.WaitForUpdateCountAsync(2, WaitTimeout);
+
+        Assert.Equal(
+            sessions.UpdateCommands[0].WatchList[0].AlarmFullReference,
+            sessions.UpdateCommands[1].WatchList[0].AlarmFullReference);
+
+        await cts.CancelAsync();
+        await monitor.StopAsync(CancellationToken.None);
+    }
+
     /// <summary>Reads the provider-status frame every stream opens with, then closes the stream.</summary>
     /// <param name="monitor">The monitor to read from.</param>
     /// <returns>The provider status carried by the first frame.</returns>
@@ -786,12 +846,14 @@ public sealed class GatewayAlarmMonitorProviderModeTests
     private static GatewayAlarmMonitor CreateMonitor(
         FakeSessionManager sessions,
         GatewayMetrics metrics,
-        IAlarmWatchListResolver resolver)
+        IAlarmWatchListResolver resolver,
+        int reconcileIntervalSeconds = 30)
     {
         AlarmsOptions options = new()
         {
             Enabled = true,
             SubscriptionExpression = @"\\NODE\Galaxy!Area",
+            ReconcileIntervalSeconds = reconcileIntervalSeconds,
         };
         return new GatewayAlarmMonitor(
             sessions,
@@ -923,11 +985,34 @@ public sealed class GatewayAlarmMonitorProviderModeTests
         /// <summary>The most recent SubscribeAlarms command the monitor sent.</summary>
         public SubscribeAlarmsCommand? LastSubscribeCommand { get; private set; }
 
+        /// <summary>Gets each live replacement command in invocation order.</summary>
+        public List<UpdateAlarmWatchListCommand> UpdateCommands { get; } = [];
+
+        /// <summary>Scripted protocol statuses for replacement commands.</summary>
+        public Queue<ProtocolStatusCode> UpdateReplyCodes { get; } = new();
+
         /// <summary>Pushes a worker event onto the monitor's event stream.</summary>
         /// <param name="mxEvent">The worker event to push.</param>
         public void EmitEvent(MxEvent mxEvent) =>
             _events.Writer.TryWrite(new WorkerEvent { Event = mxEvent });
 
+        /// <summary>Pushes a provider-mode change onto the worker event stream.</summary>
+        public void EmitModeChange(AlarmProviderMode mode, string reason) => EmitEvent(new MxEvent
+        {
+            OnAlarmProviderModeChanged = new OnAlarmProviderModeChangedEvent
+            {
+                Mode = mode,
+                Reason = reason,
+                At = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
+            },
+        });
+
+        /// <summary>Waits until at least the requested number of update commands have arrived.</summary>
+        public async Task WaitForUpdateCountAsync(int count, TimeSpan timeout)
+        {
+            await WaitUntilAsync(() => UpdateCommands.Count >= count, timeout);
+        }
+
         /// <summary>Completes once the monitor has issued its SubscribeAlarms command.</summary>
         /// <param name="timeout">The maximum time to wait.</param>
         /// <returns>A task that represents the asynchronous operation.</returns>
@@ -972,9 +1057,20 @@ public sealed class GatewayAlarmMonitorProviderModeTests
                 _subscribed.TrySetResult();
             }
 
+            if (command.Command?.Kind == MxCommandKind.UpdateAlarmWatchList)
+            {
+                UpdateCommands.Add(command.Command.UpdateAlarmWatchList.Clone());
+            }
+
             MxCommandReply reply = new()
             {
-                ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
+                ProtocolStatus = new ProtocolStatus
+                {
+                    Code = command.Command?.Kind == MxCommandKind.UpdateAlarmWatchList
+                        && UpdateReplyCodes.TryDequeue(out ProtocolStatusCode updateCode)
+                            ? updateCode
+                            : ProtocolStatusCode.Ok,
+                },
             };
             if (command.Command?.Kind == MxCommandKind.QueryActiveAlarms)
             {

src/ZB.MOM.WW.MxGateway.Tests/Contracts/ProtobufContractRoundTripTests.cs

@@ -1507,6 +1507,52 @@ public sealed class ProtobufContractRoundTripTests
         Assert.Equal(5, parsed.Failover.FailbackStableProbes);
     }
 
+    /// <summary>
+    ///     Verifies the live alarm watch-list replacement command and the
+    ///     subscription's explicit empty-list fallback intent retain their
+    ///     command kind, oneof arm, and payload across protobuf serialization.
+    /// </summary>
+    [Fact]
+    public void AlarmWatchListCommands_RoundTripFallbackIntentAndReplacementPayload()
+    {
+        var subscribe = new SubscribeAlarmsCommand
+        {
+            SubscriptionExpression = @"\\node\Galaxy!Area",
+            FallbackEnabled = true,
+        };
+        var parsedSubscribe = SubscribeAlarmsCommand.Parser.ParseFrom(subscribe.ToByteArray());
+
+        Assert.True(parsedSubscribe.FallbackEnabled);
+        Assert.Empty(parsedSubscribe.WatchList);
+
+        var original = new MxCommand
+        {
+            Kind = MxCommandKind.UpdateAlarmWatchList,
+            UpdateAlarmWatchList = new UpdateAlarmWatchListCommand
+            {
+                WatchList =
+                {
+                    new AlarmSubtagTarget
+                    {
+                        AlarmFullReference = "Galaxy!Area.Tank01.Level.HiHi",
+                        SourceObjectReference = "Tank01",
+                        ActiveSubtag = "Tank01.Level.HiHi.InAlarm",
+                        AckedSubtag = "Tank01.Level.HiHi.Acked",
+                        AckCommentSubtag = "Tank01.Level.HiHi.AckMsg",
+                        PrioritySubtag = "Tank01.Level.HiHi.Priority",
+                    },
+                },
+            },
+        };
+
+        var parsed = MxCommand.Parser.ParseFrom(original.ToByteArray());
+
+        Assert.Equal(original, parsed);
+        Assert.Equal(MxCommandKind.UpdateAlarmWatchList, parsed.Kind);
+        Assert.Equal(MxCommand.PayloadOneofCase.UpdateAlarmWatchList, parsed.PayloadCase);
+        Assert.Single(parsed.UpdateAlarmWatchList.WatchList);
+    }
+
     /// <summary>
     ///     Verifies that an <see cref="MxEvent"/> carrying an
     ///     <see cref="OnAlarmProviderModeChangedEvent"/> body (the