feat(GWC-37,WRK-32): refresh live alarm watch lists

src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs

@@ -66,9 +66,10 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
     private bool _subtagCoverageMissing;
     private string _subtagCoverageReason = string.Empty;
 
-    // The watch-list references actually sent to the worker in SubscribeAlarms, guarded by _sync.
-    // The worker holds that list for the life of the subscription, so a later re-resolution is
-    // compared against this to report drift rather than silently diverging.
+    // The full canonical watch-list fingerprint acknowledged by the worker and its alarm
+    // references, guarded by _sync. The fingerprint advances only after an OK subscribe/update
+    // reply, so a rejected replacement remains material and is retried on the next refresh edge.
+    private string _acknowledgedWatchListFingerprint = string.Empty;
     private IReadOnlyCollection<string> _sentWatchListReferences = [];
 
     // Whether the worker's most recent reconcile fetch was capped, guarded by _sync.
@@ -356,9 +357,6 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
         // feed subscribers outlive the monitor's worker session and may already be attached.
         lock (_sync)
         {
-            _sentWatchListReferences = watchList
-                .Select(target => target.AlarmFullReference)
-                .ToHashSet(StringComparer.OrdinalIgnoreCase);
             SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false), broadcast: true);
         }
 
@@ -374,6 +372,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
         {
             SubscriptionExpression = subscription,
             ForcedMode = forcedMode,
+            FallbackEnabled = CanUseSubtagFallback(forcedMode),
             Failover = new AlarmFailoverConfig
             {
                 ConsecutiveFailureThreshold = _options.Fallback.ConsecutiveFailureThreshold,
@@ -404,6 +403,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
                 ?? $"status {code}";
             throw new InvalidOperationException($"Worker rejected SubscribeAlarms: {diagnostic}");
         }
+
+        lock (_sync)
+        {
+            AcknowledgeWatchList(watchList);
+        }
     }
 
     private async Task ReconcileLoopAsync(string sessionId, CancellationToken cancellationToken)
@@ -416,6 +420,8 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
             {
                 try
                 {
+                    await RefreshAlarmWatchListAsync(sessionId, broadcast: true, cancellationToken)
+                        .ConfigureAwait(false);
                     await ReconcileAsync(sessionId, cancellationToken).ConfigureAwait(false);
                 }
                 catch (OperationCanceledException)
@@ -545,18 +551,14 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
         AlarmProviderMode toMode = change.Mode;
         string reason = change.Reason ?? string.Empty;
 
-        // GWC-37: this is the moment the subtag watch-list stops being theoretical — the provider
-        // that is about to serve is the one built from it. Re-resolve BEFORE the status is
-        // published so the frame the operator sees on the failover carries the coverage verdict
-        // rather than a second frame later, and so a watch-list that has drifted since
-        // SubscribeAlarms is reported instead of silently diverging. The re-resolution is NOT
-        // re-sent: the worker rejects a second SubscribeAlarms while subscribed, and the only way
-        // to replace the list — Unsubscribe then Subscribe — rebuilds the FailoverAlarmConsumer,
-        // which restarts on the primary that just failed and discards the warm standby state, so
-        // it would flap the feed rather than repair it. A monitor restart re-resolves and re-sends.
+        // The failover edge is the moment the subtag watch-list stops being theoretical. Refresh
+        // it before publishing the status frame so recovered discovery is applied to the already
+        // active standby and the operator sees the acknowledged coverage verdict atomically with
+        // the provider switch.
         if (toMode == AlarmProviderMode.Subtag)
         {
-            await RefreshSubtagCoverageAsync(cancellationToken).ConfigureAwait(false);
+            await RefreshAlarmWatchListAsync(sessionId, broadcast: false, cancellationToken)
+                .ConfigureAwait(false);
         }
 
         AlarmProviderStatus status;
@@ -606,14 +608,17 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
         }
     }
 
-    // GWC-37: re-resolves the subtag watch-list and re-grades the coverage verdict without
-    // touching the worker's subscription. Best-effort: a resolver that throws (only cancellation
-    // and programming errors do — discovery failure is reported in the resolution) leaves the
-    // verdict from the last subscribe in place, because a stale verdict is better than none.
-    // Does not broadcast: the callers publish a status frame of their own immediately after.
-    private async Task RefreshSubtagCoverageAsync(CancellationToken cancellationToken)
+    private async Task RefreshAlarmWatchListAsync(
+        string sessionId,
+        bool broadcast,
+        CancellationToken cancellationToken)
     {
         AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode);
+        if (!CanUseSubtagFallback(forcedMode))
+        {
+            return;
+        }
+
         AlarmWatchListResolution resolution;
         try
         {
@@ -627,18 +632,96 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
         {
             _logger.LogWarning(
                 exception,
-                "Re-resolving the alarm subtag watch-list failed; keeping the coverage verdict from the last subscribe.");
+                "Re-resolving the alarm subtag watch-list failed; keeping the last acknowledged watch list.");
             return;
         }
 
+        string resolvedFingerprint = CanonicalWatchListFingerprint(resolution.Targets);
+        string acknowledgedFingerprint;
         lock (_sync)
         {
-            bool stale = !resolution.Targets
-                .Select(target => target.AlarmFullReference)
-                .ToHashSet(StringComparer.OrdinalIgnoreCase)
-                .SetEquals(_sentWatchListReferences);
-            SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, stale), broadcast: false);
+            acknowledgedFingerprint = _acknowledgedWatchListFingerprint;
         }
+
+        if (string.Equals(resolvedFingerprint, acknowledgedFingerprint, StringComparison.Ordinal))
+        {
+            lock (_sync)
+            {
+                SetSubtagCoverage(
+                    GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false),
+                    broadcast);
+            }
+            return;
+        }
+
+        UpdateAlarmWatchListCommand update = new();
+        update.WatchList.AddRange(resolution.Targets);
+        WorkerCommandReply reply = await _sessionManager.InvokeAsync(
+                sessionId,
+                new WorkerCommand
+                {
+                    Command = new MxCommand
+                    {
+                        Kind = MxCommandKind.UpdateAlarmWatchList,
+                        UpdateAlarmWatchList = update,
+                    },
+                },
+                cancellationToken)
+            .ConfigureAwait(false);
+
+        ProtocolStatusCode? code = reply.Reply?.ProtocolStatus?.Code;
+        lock (_sync)
+        {
+            if (code == ProtocolStatusCode.Ok)
+            {
+                AcknowledgeWatchList(resolution.Targets);
+                SetSubtagCoverage(
+                    GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false),
+                    broadcast);
+                return;
+            }
+
+            string diagnostic = reply.Reply?.DiagnosticMessage
+                ?? reply.Reply?.ProtocolStatus?.Message
+                ?? $"status {code}";
+            string reason = GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: true);
+            SetSubtagCoverage($"{reason}; worker rejected watch-list update ({diagnostic})", broadcast);
+        }
+    }
+
+    private bool CanUseSubtagFallback(AlarmProviderMode forcedMode) =>
+        forcedMode != AlarmProviderMode.Alarmmgr
+        && (forcedMode == AlarmProviderMode.Subtag
+            || _options.Fallback.Discovery.UseGalaxyRepository
+            || _options.Fallback.Discovery.IncludeAttributes.Length > 0);
+
+    // Caller holds _sync.
+    private void AcknowledgeWatchList(IReadOnlyList<AlarmSubtagTarget> watchList)
+    {
+        _acknowledgedWatchListFingerprint = CanonicalWatchListFingerprint(watchList);
+        _sentWatchListReferences = watchList
+            .Select(target => target.AlarmFullReference)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+    }
+
+    private static string CanonicalWatchListFingerprint(IEnumerable<AlarmSubtagTarget> watchList) =>
+        string.Join(
+            "\n",
+            watchList
+                .Select(target => string.Join(
+                    "|",
+                    FingerprintPart(target.AlarmFullReference),
+                    FingerprintPart(target.SourceObjectReference),
+                    FingerprintPart(target.ActiveSubtag),
+                    FingerprintPart(target.AckedSubtag),
+                    FingerprintPart(target.AckCommentSubtag),
+                    FingerprintPart(target.PrioritySubtag)))
+                .Order(StringComparer.Ordinal));
+
+    private static string FingerprintPart(string? value)
+    {
+        string normalized = value ?? string.Empty;
+        return $"{normalized.Length}:{normalized}";
     }
 
     // Grades one resolution against the effective mode. Returns the reason the subtag coverage is
@@ -684,8 +767,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
         {
             reasons.Add(
                 $"{AlarmProviderReasons.SubtagWatchListStale} ({resolution.Targets.Count} target(s) now "
-                + $"resolve against {_sentWatchListReferences.Count} sent at subscribe); restart the alarm "
-                + "monitor to re-send it");
+                + $"resolve against {_sentWatchListReferences.Count} acknowledged by the worker)");
         }
 
         return string.Join("; ", reasons);