diff --git a/docs/AlarmClientDiscovery.md b/docs/AlarmClientDiscovery.md index 411a500..451c1d5 100644 --- a/docs/AlarmClientDiscovery.md +++ b/docs/AlarmClientDiscovery.md @@ -835,7 +835,7 @@ carries `degraded = true`. ### Watch-list discovery `GatewayAlarmMonitor` resolves the subtag watch-list at subscribe time by -calling `IAlarmWatchListResolver.GetAlarmAttributesAsync`. The resolver merges: +calling `IAlarmWatchListResolver.ResolveAsync`. The resolver merges: 1. Galaxy Repository SQL (`GetAlarmAttributesAsync`) — objects that have alarm extensions in the configured area. @@ -847,8 +847,42 @@ The resolved list is a set of `AlarmSubtagTarget` messages sent to the worker inside `SubscribeAlarmsCommand.watch_list`. Each target carries the composed MXAccess item addresses for the `InAlarm`, `Acked`, `AckMsg`, and `Priority` subtags (confirmed AVEVA `AlarmExtension` field names, verified against the live -ZB Galaxy `attribute_definition` rows). The gateway re-runs discovery on its -reconcile cadence and pushes an updated watch-list when the model changes. +ZB Galaxy `attribute_definition` rows). + +**The list is fixed for the life of the subscription.** The worker builds its +subtag consumer from the targets `SubscribeAlarms` carried and keeps them until +the subscription ends; a second `SubscribeAlarms` on a subscribed session is +rejected, and the only way to replace the list — `UnsubscribeAlarms` then +`SubscribeAlarms` — rebuilds `FailoverAlarmConsumer`, restarting it on the +primary and discarding the warm standby state. Re-sending on a failover would +therefore flap the feed rather than repair it. A monitor restart (a worker fault, +or a gateway restart) re-resolves and re-sends, and that is the supported way to +pick up Galaxy objects added since the monitor started. + +Because the list is fixed, the resolution is **graded** rather than merely +logged (GWC-37). `ResolveAsync` returns an `AlarmWatchListResolution` carrying +whether Galaxy Repository discovery was attempted and whether it failed, and the +monitor folds the verdict into the alarm feed's `AlarmProviderStatus`: + +- Zero targets while the effective mode can need them (`Auto` or `ForceSubtag`) + sets `degraded = true` with reason `AlarmProviderReasons.SubtagWatchListEmpty` + — the subtag provider covers no alarms. +- Discovery enabled but failing adds + `AlarmProviderReasons.SubtagDiscoveryFailed` and the failure message, so a + Galaxy Repository outage at startup is a standing status rather than one log + line emitted possibly weeks before the failover that needs the list. +- On the failover edge (`ApplyProviderModeChangeAsync` with `toMode == Subtag`) + the monitor re-resolves and compares against what it sent. A difference is + reported as `AlarmProviderReasons.SubtagWatchListStale` with both counts, on + the same status frame as the failover, because it cannot be applied. + +`degraded` on the status is therefore the OR of "the subtag provider is the one +serving" and "the subtag watch-list cannot cover anything". In `Auto` the second +can be true while the mode is still `ALARMMGR`: the live feed is healthy and the +standby behind it is empty. The dashboard renders that combination as the amber +`Alarm Manager (no subtag standby)` badge rather than the failover label, so the +badge never names the wrong live provider. `ForceAlarmManager` never builds a +subtag consumer, so its watch-list is not graded at all. Each target's canonical `AlarmFullReference` is composed as `Galaxy!{area}.{reference}` (literal `Galaxy` provider). The `{area}` is the diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 5970259..bf4b6e8 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -445,11 +445,11 @@ native wnwrap alarm-manager provider and the subtag-monitoring fallback. | Option | Default | Description | |--------|---------|-------------| -| `MxGateway:Alarms:Fallback:Mode` | `Auto` | Provider selection mode. `Auto` uses the alarm manager as primary and fails over to subtag monitoring after consecutive COM failures, then fails back automatically. `ForceAlarmManager` disables failover. `ForceSubtag` forces subtag monitoring on from startup. Values are case-insensitive. | +| `MxGateway:Alarms:Fallback:Mode` | `Auto` | Provider selection mode. `Auto` uses the alarm manager as primary and fails over to subtag monitoring after consecutive COM failures, then fails back automatically. `ForceAlarmManager` disables failover (and is the one mode whose subtag watch-list is never graded, because no subtag consumer is built). `ForceSubtag` forces subtag monitoring on from startup. Values are case-insensitive. | | `MxGateway:Alarms:Fallback:ConsecutiveFailureThreshold` | `3` | Number of consecutive wnwrap COM failures (`COMException` or failure HRESULT from `Subscribe` / `GetXmlCurrentAlarms2`) before the monitor switches to subtag mode. Floored at 1. | | `MxGateway:Alarms:Fallback:FailbackProbeIntervalSeconds` | `30` | While in subtag mode, how often (in seconds) the monitor probes the wnwrap provider to detect recovery. Floored at 1. | | `MxGateway:Alarms:Fallback:FailbackStableProbes` | `3` | Number of consecutive clean wnwrap probes required before the monitor switches back to the alarm manager. Floored at 1. | -| `MxGateway:Alarms:Fallback:Discovery:UseGalaxyRepository` | `true` | When `true`, the monitor queries the Galaxy Repository SQL database to build the subtag watch-list for the configured area. | +| `MxGateway:Alarms:Fallback:Discovery:UseGalaxyRepository` | `true` | When `true`, the monitor queries the Galaxy Repository SQL database to build the subtag watch-list for the configured area. A failed query is not fatal — the monitor continues with whatever configuration contributed — but with `Mode` `Auto` or `ForceSubtag` the resulting coverage gap is reported on the alarm feed's `AlarmProviderStatus` (`degraded = true` with a `Subtag watch-list is empty` / `Galaxy Repository alarm discovery failed` reason) and shown on the dashboard as the amber `Alarm Manager (no subtag standby)` badge, instead of only a startup log line. | | `MxGateway:Alarms:Fallback:Discovery:Area` | _(empty)_ | Galaxy area to scope the Repository query to. Falls back to `MxGateway:Alarms:DefaultArea` when empty. Ignored when `UseGalaxyRepository` is `false`. This area is **not** used to compose a Repository-derived alarm's canonical `Galaxy!{area}.{reference}`: each discovered alarm uses its object's real Galaxy area (discovered via `gobject.area_gobject_id`), so the reference's group matches what the native alarmmgr emits. `Discovery:Area` / `DefaultArea` is used as the composition area only for explicit `IncludeAttributes` entries, which carry no discovered area. | | `MxGateway:Alarms:Fallback:Discovery:IncludeAttributes` | _(empty)_ | Explicit MXAccess attribute paths to add to the subtag watch-list, supplementing (or replacing, when `UseGalaxyRepository` is `false`) the Repository-derived list. | | `MxGateway:Alarms:Fallback:Discovery:ExcludeAttributes` | _(empty)_ | Attribute paths to remove from the merged watch-list (case-insensitive). The exclude is applied after the Repository-derived rows and the explicit `IncludeAttributes` entries are combined, so an exclude that matches an explicit include suppresses it too — excludes win. Ignored when `UseGalaxyRepository` is `false`. | diff --git a/gateway.md b/gateway.md index 77c113f..7cc7912 100644 --- a/gateway.md +++ b/gateway.md @@ -249,6 +249,22 @@ monitoring (forced)") when subtag mode is the configured `Fallback:Mode=ForceSub as a fault. Metrics: `mxgateway.alarms.provider_mode` gauge (1 = alarmmgr, 2 = subtag) and `mxgateway.alarms.provider_switches` counter. +**Standby-coverage visibility:** `degraded` also covers the case where the alarm +manager is serving normally but the subtag standby behind it could not cover +anything — a watch-list that resolved to zero targets, or Galaxy Repository +discovery that was enabled and failed. That condition used to be a single startup +log line, invisible until the failover that needed the standby found it empty; it +is now a standing `AlarmProviderStatus` reason +(`AlarmProviderReasons.SubtagWatchListEmpty` / `SubtagDiscoveryFailed`) carried on +the feed and rendered as the amber `Alarm Manager (no subtag standby)` badge, which +keeps the failover label from naming the wrong live provider. The watch-list the +worker holds is fixed for the life of the subscription, so on the failover edge the +monitor re-resolves and reports a difference +(`AlarmProviderReasons.SubtagWatchListStale`, with both counts) rather than +re-sending it — re-sending would rebuild the failover consumer on the primary that +just failed. A monitor restart re-resolves and re-sends. See +`docs/AlarmClientDiscovery.md`. + **Truncated-snapshot visibility:** `GetXmlCurrentAlarms2` caps its reply at `MxGateway:Alarms:MaxAlarmsPerFetch` and offers no confirmed "more available" flag, so a reply holding exactly the cap is treated as truncated. On such a diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/TestSupport/EmptyAlarmWatchListResolver.cs b/src/ZB.MOM.WW.MxGateway.IntegrationTests/TestSupport/EmptyAlarmWatchListResolver.cs index 332a81a..2ee88e5 100644 --- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/TestSupport/EmptyAlarmWatchListResolver.cs +++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/TestSupport/EmptyAlarmWatchListResolver.cs @@ -1,4 +1,3 @@ -using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Alarms; using ZB.MOM.WW.MxGateway.Server.Configuration; @@ -19,14 +18,18 @@ public sealed class EmptyAlarmWatchListResolver : IAlarmWatchListResolver /// Shared no-op instance. public static readonly EmptyAlarmWatchListResolver Instance = new(); - private static readonly IReadOnlyList Empty = []; + private static readonly AlarmWatchListResolution Empty = new( + [], + DiscoveryAttempted: false, + DiscoveryFailed: false, + DiscoveryFailureMessage: null); private EmptyAlarmWatchListResolver() { } /// - public Task> ResolveAsync( + public Task ResolveAsync( AlarmsOptions options, CancellationToken cancellationToken = default) { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmProviderReasons.cs b/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmProviderReasons.cs index 045f3aa..46359b2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmProviderReasons.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmProviderReasons.cs @@ -15,4 +15,32 @@ public static class AlarmProviderReasons /// distinguish a forced subtag mode from an unexpected degraded failover. /// public const string ForcedSubtag = "Forced subtag mode (configuration)"; + + /// + /// Reason prefix set when the effective mode can need the subtag watch-list + /// (Auto or ForceSubtag) but the list resolved to zero targets, so the + /// subtag provider covers no alarms (GWC-37). In Auto this is a warning about the + /// standby, not about the currently-serving alarm manager, which is why the dashboard + /// renders it against the ALARMMGR mode as a coverage badge rather than a failover. + /// + public const string SubtagWatchListEmpty = + "Subtag watch-list is empty: the subtag provider covers no alarms"; + + /// + /// Reason prefix set when Galaxy Repository alarm discovery was enabled but failed, so + /// the watch-list carries only what configuration contributed and may be incomplete + /// (GWC-37). Distinct from : discovery failing with a + /// usable config-only list is a completeness warning, not a total loss of coverage. + /// + public const string SubtagDiscoveryFailed = + "Galaxy Repository alarm discovery failed: the subtag watch-list may be incomplete"; + + /// + /// Reason prefix set when a re-resolution at the failover edge produced a different + /// watch-list than the one sent at SubscribeAlarms (GWC-37). The worker keeps the + /// list it was given for the life of the subscription, so the difference is reported + /// rather than applied; a monitor restart re-resolves and re-sends. + /// + public const string SubtagWatchListStale = + "Subtag watch-list differs from the one the worker is using"; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolution.cs b/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolution.cs new file mode 100644 index 0000000..8c19eef --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolution.cs @@ -0,0 +1,29 @@ +using ZB.MOM.WW.MxGateway.Contracts.Proto; + +namespace ZB.MOM.WW.MxGateway.Server.Alarms; + +/// +/// The outcome of one subtag watch-list resolution: the composed targets plus enough +/// provenance for the caller to tell an empty list that means "nothing was configured" +/// from one that means "discovery was configured and silently produced nothing" +/// (GWC-37). deliberately never throws +/// on a discovery failure, so without this the two are indistinguishable at the call site +/// and a Galaxy Repository outage at startup can only be found in a log line. +/// +/// The resolved watch-list, in order; possibly empty. +/// +/// Galaxy Repository discovery was enabled for this resolution +/// (Fallback:Discovery:UseGalaxyRepository). +/// +/// +/// Discovery was attempted and failed, so holds only what +/// configuration contributed. +/// +/// +/// The discovery failure's message, or null when discovery did not fail. +/// +public sealed record AlarmWatchListResolution( + IReadOnlyList Targets, + bool DiscoveryAttempted, + bool DiscoveryFailed, + string? DiscoveryFailureMessage); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolver.cs b/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolver.cs index 6844cf0..b17c053 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolver.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Alarms/AlarmWatchListResolver.cs @@ -31,7 +31,7 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver } /// - public async Task> ResolveAsync( + public async Task ResolveAsync( AlarmsOptions options, CancellationToken cancellationToken = default) { @@ -39,6 +39,11 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver AlarmDiscoveryOptions discovery = options.Fallback.Discovery; + // GWC-37: the caller cannot grade an empty watch-list without knowing whether discovery + // was asked to contribute and whether it failed while doing so, because the failure is + // deliberately swallowed below. + string? discoveryFailureMessage = null; + // Config fallback area used only for explicit IncludeAttributes entries (which // carry no discovered area): discovery area, else the default area (may be empty). string configFallbackArea = string.IsNullOrEmpty(discovery.Area) ? options.DefaultArea : discovery.Area; @@ -68,6 +73,7 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver _logger.LogWarning( ex, "Galaxy Repository alarm-attribute discovery failed; continuing with configuration-only watch-list."); + discoveryFailureMessage = ex.Message; rows = []; } @@ -148,7 +154,11 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver _logger.LogDebug("Resolved alarm subtag watch-list with {TargetCount} target(s).", targets.Count); } - return targets; + return new AlarmWatchListResolution( + targets, + DiscoveryAttempted: discovery.UseGalaxyRepository, + DiscoveryFailed: discoveryFailureMessage is not null, + discoveryFailureMessage); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs b/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs index 9058615..04fd3cb 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs @@ -57,6 +57,20 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic private string _providerReason = string.Empty; private DateTimeOffset _providerSince = DateTimeOffset.UtcNow; + // GWC-37 subtag-coverage verdict, guarded by _sync: whether the watch-list the worker is + // using can actually cover anything, and why not. Independent of _providerDegraded (which + // means "the subtag provider is the one serving"): in Auto mode the alarm manager can be + // serving perfectly while the standby behind it covers zero alarms, and that is invisible + // until the failover that needs it. Folded into BuildProviderStatus so it reaches the feed, + // the dashboard badge and every late-joining subscriber rather than only a startup log line. + private bool _subtagCoverageMissing; + private string _subtagCoverageReason = string.Empty; + + // The watch-list references actually sent to the worker in SubscribeAlarms, guarded by _sync. + // The worker holds that list for the life of the subscription, so a later re-resolution is + // compared against this to report drift rather than silently diverging. + private IReadOnlyCollection _sentWatchListReferences = []; + // Whether the worker's most recent reconcile fetch was capped, guarded by _sync. // Written only by ApplyReconcile (and cleared with the cache), so it describes the last full // reconcile — not necessarily the current _alarms contents, which live transitions keep moving @@ -329,12 +343,25 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic private async Task SubscribeAlarmsAsync(string sessionId, string subscription, CancellationToken cancellationToken) { - IReadOnlyList watchList = await _watchListResolver + AlarmWatchListResolution resolution = await _watchListResolver .ResolveAsync(_options, cancellationToken) .ConfigureAwait(false); + IReadOnlyList watchList = resolution.Targets; AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode); + // GWC-37: grade the coverage before the command goes out, so a Galaxy Repository outage at + // startup is a standing status on the feed instead of one log line an operator reads weeks + // later — when the failover that needed the watch-list finds it empty. Broadcast because + // feed subscribers outlive the monitor's worker session and may already be attached. + lock (_sync) + { + _sentWatchListReferences = watchList + .Select(target => target.AlarmFullReference) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false), broadcast: true); + } + _logger.LogInformation( "Alarm subscribe: forcedMode={ForcedMode} configMode={ConfigMode} watchList={WatchListCount}.", forcedMode, _options.Fallback.Mode, watchList.Count); @@ -518,6 +545,20 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic AlarmProviderMode toMode = change.Mode; string reason = change.Reason ?? string.Empty; + // GWC-37: this is the moment the subtag watch-list stops being theoretical — the provider + // that is about to serve is the one built from it. Re-resolve BEFORE the status is + // published so the frame the operator sees on the failover carries the coverage verdict + // rather than a second frame later, and so a watch-list that has drifted since + // SubscribeAlarms is reported instead of silently diverging. The re-resolution is NOT + // re-sent: the worker rejects a second SubscribeAlarms while subscribed, and the only way + // to replace the list — Unsubscribe then Subscribe — rebuilds the FailoverAlarmConsumer, + // which restarts on the primary that just failed and discards the warm standby state, so + // it would flap the feed rather than repair it. A monitor restart re-resolves and re-sends. + if (toMode == AlarmProviderMode.Subtag) + { + await RefreshSubtagCoverageAsync(cancellationToken).ConfigureAwait(false); + } + AlarmProviderStatus status; int fromModeInt; lock (_sync) @@ -565,14 +606,141 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic } } + // GWC-37: re-resolves the subtag watch-list and re-grades the coverage verdict without + // touching the worker's subscription. Best-effort: a resolver that throws (only cancellation + // and programming errors do — discovery failure is reported in the resolution) leaves the + // verdict from the last subscribe in place, because a stale verdict is better than none. + // Does not broadcast: the callers publish a status frame of their own immediately after. + private async Task RefreshSubtagCoverageAsync(CancellationToken cancellationToken) + { + AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode); + AlarmWatchListResolution resolution; + try + { + resolution = await _watchListResolver.ResolveAsync(_options, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + _logger.LogWarning( + exception, + "Re-resolving the alarm subtag watch-list failed; keeping the coverage verdict from the last subscribe."); + return; + } + + lock (_sync) + { + bool stale = !resolution.Targets + .Select(target => target.AlarmFullReference) + .ToHashSet(StringComparer.OrdinalIgnoreCase) + .SetEquals(_sentWatchListReferences); + SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, stale), broadcast: false); + } + } + + // Grades one resolution against the effective mode. Returns the reason the subtag coverage is + // unsound, or an empty string when it is sound (or irrelevant, as under ForceAlarmManager). + // Empty targets are the total-loss case; a failed discovery with a usable config-only list is + // a completeness warning; drift is reported separately because the worker cannot be updated. + private string GradeSubtagCoverage( + AlarmProviderMode forcedMode, + AlarmWatchListResolution resolution, + bool staleAgainstSent) + { + // ForceAlarmManager never builds a subtag consumer, so its watch-list is genuinely unused. + if (forcedMode == AlarmProviderMode.Alarmmgr) + { + return string.Empty; + } + + // Nothing was asked of discovery and nothing was configured: subtag fallback is simply not + // in use in this deployment (Auto with an empty list subscribes alarmmgr-only), so there is + // no coverage to warn about. + if (resolution.Targets.Count == 0 + && !resolution.DiscoveryAttempted + && _options.Fallback.Discovery.IncludeAttributes.Length == 0 + && forcedMode != AlarmProviderMode.Subtag) + { + return string.Empty; + } + + List reasons = []; + if (resolution.Targets.Count == 0) + { + reasons.Add(AlarmProviderReasons.SubtagWatchListEmpty); + } + + if (resolution.DiscoveryFailed) + { + reasons.Add(resolution.DiscoveryFailureMessage is { Length: > 0 } message + ? $"{AlarmProviderReasons.SubtagDiscoveryFailed} ({message})" + : AlarmProviderReasons.SubtagDiscoveryFailed); + } + + if (staleAgainstSent) + { + reasons.Add( + $"{AlarmProviderReasons.SubtagWatchListStale} ({resolution.Targets.Count} target(s) now " + + $"resolve against {_sentWatchListReferences.Count} sent at subscribe); restart the alarm " + + "monitor to re-send it"); + } + + return string.Join("; ", reasons); + } + + // Caller holds _sync. Applies a coverage verdict, logging and (optionally) broadcasting only + // when it actually changed so a stable degraded state does not repeat a frame every pass. + private void SetSubtagCoverage(string reason, bool broadcast) + { + bool missing = reason.Length > 0; + if (missing == _subtagCoverageMissing && string.Equals(reason, _subtagCoverageReason, StringComparison.Ordinal)) + { + return; + } + + _subtagCoverageMissing = missing; + _subtagCoverageReason = reason; + + if (missing) + { + _logger.LogWarning("Alarm subtag coverage is unsound: {Reason}", reason); + } + else + { + _logger.LogInformation("Alarm subtag coverage is sound."); + } + + if (broadcast) + { + BroadcastToAll(new AlarmFeedMessage { ProviderStatus = BuildProviderStatus() }); + } + } + // Caller holds _sync. Builds an AlarmProviderStatus snapshot of the current state. + // + // GWC-37: degraded is the OR of "the subtag provider is the one serving" and "the subtag + // watch-list cannot cover anything". The second is what makes an empty watch-list visible in + // Auto mode, where the alarm manager is serving happily and the standby behind it is empty. + // The reasons are concatenated when both apply, which is the worst case worth naming loudly: + // failed over to a provider that covers nothing. private AlarmProviderStatus BuildProviderStatus() { + bool degraded = _providerDegraded || _subtagCoverageMissing; + string reason = (_providerReason, _subtagCoverageReason) switch + { + ("", var coverage) => coverage, + (var mode, "") => mode, + (var mode, var coverage) => $"{mode} — {coverage}", + }; + return new AlarmProviderStatus { Mode = _providerMode, - Degraded = _providerDegraded, - Reason = _providerReason, + Degraded = degraded, + Reason = reason, Since = Timestamp.FromDateTimeOffset(_providerSince), }; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Alarms/IAlarmWatchListResolver.cs b/src/ZB.MOM.WW.MxGateway.Server/Alarms/IAlarmWatchListResolver.cs index 196b2b8..55dc2d4 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Alarms/IAlarmWatchListResolver.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Alarms/IAlarmWatchListResolver.cs @@ -18,13 +18,15 @@ public interface IAlarmWatchListResolver /// Alarm configuration carrying discovery and subtag-name settings. /// Token to cancel the asynchronous operation. /// - /// The resolved watch-list, possibly empty. + /// The resolved watch-list, possibly empty, together + /// with the discovery provenance the caller needs to grade it. /// Discovery being unavailable never throws — it yields an empty (or - /// config-only) list and the caller decides what to do with it. Cancellation + /// config-only) list with set, + /// and the caller decides what to do with it. Cancellation /// is the one exception: a triggered /// still propagates an . /// - Task> ResolveAsync( + Task ResolveAsync( AlarmsOptions options, CancellationToken cancellationToken = default); } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAlarmProviderStatus.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAlarmProviderStatus.cs index de035b1..19e01b9 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAlarmProviderStatus.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAlarmProviderStatus.cs @@ -30,6 +30,14 @@ public sealed record DashboardAlarmProviderStatus( /// public const string ForcedSubtagLabel = "Subtag monitoring (forced)"; + /// + /// Badge label shown when the alarm manager is serving normally but the subtag standby + /// behind it covers nothing — an empty or undiscoverable watch-list (GWC-37). The feed is + /// healthy today; what is degraded is the fallback that would take over tomorrow, so the + /// badge names the standby rather than claiming the feed has failed over. + /// + public const string NoSubtagStandbyLabel = "Alarm Manager (no subtag standby)"; + private const string HealthyBadge = "bg-success"; private const string DegradedBadge = "bg-warning text-dark"; @@ -69,8 +77,18 @@ public sealed record DashboardAlarmProviderStatus( && status.Mode == AlarmProviderMode.Subtag && string.Equals(reason, AlarmProviderReasons.ForcedSubtag, StringComparison.Ordinal); - string label = !degraded ? AlarmManagerLabel : forced ? ForcedSubtagLabel : DegradedLabel; - string badge = !degraded ? HealthyBadge : forced ? ForcedSubtagBadge : DegradedBadge; + // GWC-37: degraded WITHOUT subtag mode means the alarm manager is still serving and the + // standby is the unsound half. "Subtag monitoring (degraded)" would misreport which + // provider is live, so this state gets its own label; the amber badge is shared because + // the operator action — fix discovery — is equally overdue in both. + bool standbyOnlyWarning = degraded && status.Mode != AlarmProviderMode.Subtag; + + string label = !degraded + ? AlarmManagerLabel + : standbyOnlyWarning ? NoSubtagStandbyLabel : forced ? ForcedSubtagLabel : DegradedLabel; + string badge = !degraded + ? HealthyBadge + : standbyOnlyWarning ? DegradedBadge : forced ? ForcedSubtagBadge : DegradedBadge; return new DashboardAlarmProviderStatus( Mode: status.Mode, diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs index bf9a0d0..3a11ff9 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs @@ -384,9 +384,14 @@ public sealed class AlarmFailoverEndToEndTests private sealed class StubWatchListResolver(IReadOnlyList targets) : IAlarmWatchListResolver { /// - public Task> ResolveAsync( + public Task ResolveAsync( AlarmsOptions options, - CancellationToken cancellationToken = default) => Task.FromResult(targets); + CancellationToken cancellationToken = default) => + Task.FromResult(new AlarmWatchListResolution( + targets, + DiscoveryAttempted: false, + DiscoveryFailed: false, + DiscoveryFailureMessage: null)); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs index b226a22..6aa2177 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs @@ -507,10 +507,14 @@ public sealed class AlarmTruncationSignalTests private sealed class StubWatchListResolver : IAlarmWatchListResolver { /// - public Task> ResolveAsync( + public Task ResolveAsync( AlarmsOptions options, CancellationToken cancellationToken = default) => - Task.FromResult>([]); + Task.FromResult(new AlarmWatchListResolution( + [], + DiscoveryAttempted: false, + DiscoveryFailed: false, + DiscoveryFailureMessage: null)); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmWatchListResolverTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmWatchListResolverTests.cs index f8c8b1b..62f8809 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmWatchListResolverTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmWatchListResolverTests.cs @@ -16,6 +16,21 @@ public sealed class AlarmWatchListResolverTests private static AlarmWatchListResolver CreateResolver(IGalaxyRepository repository) => new(repository, NullLogger.Instance); + /// + /// Resolves and projects to the target list. These cases assert composition rules, which + /// live in ; the discovery-provenance fields + /// the resolution also carries (GWC-37) are asserted by their own cases below. + /// + /// The resolver under test. + /// Alarm options to resolve against. + /// Token to cancel the resolution. + /// The resolved targets. + private static async Task> ResolveTargetsAsync( + AlarmWatchListResolver resolver, + AlarmsOptions options, + CancellationToken cancellationToken = default) => + (await resolver.ResolveAsync(options, cancellationToken)).Targets; + private static AlarmsOptions Options( bool useGalaxyRepository = true, string area = "", @@ -58,7 +73,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( include: ["pump01.fault", "Valve03.Position.Lo"], exclude: ["Tank02.Level.HiHi"])); @@ -81,7 +96,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( subtags: new AlarmSubtagNameOptions { Active = "InAlarm", @@ -110,7 +125,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); // Default Priority is "Priority"; force it empty alongside empty AckComment. - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( subtags: new AlarmSubtagNameOptions { Priority = string.Empty, AckComment = string.Empty })); AlarmSubtagTarget target = Assert.Single(result); @@ -140,7 +155,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); // Config area "DEV" must be ignored for a GR row that has a discovered area. - IReadOnlyList result = await resolver.ResolveAsync( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options(area: "DEV", defaultArea: "DEV")); AlarmSubtagTarget target = Assert.Single(result); @@ -161,7 +176,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); // No discovery area and no default area. - IReadOnlyList result = await resolver.ResolveAsync(Options()); + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options()); AlarmSubtagTarget target = Assert.Single(result); Assert.Equal("Galaxy!Tank01.Level.HiHi", target.AlarmFullReference); @@ -178,7 +193,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( area: "Site_A", include: ["Tank01.Level.HiHi"])); @@ -196,7 +211,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( area: "", defaultArea: "Plant", include: ["Tank01.Level.HiHi"])); @@ -217,7 +232,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( useGalaxyRepository: false, include: ["Tank01.Level.HiHi"])); @@ -235,7 +250,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( include: ["Tank01.Level.HiHi"])); // Did not throw; discovery set empty, include retained. @@ -243,6 +258,51 @@ public sealed class AlarmWatchListResolverTests Assert.Equal("Galaxy!Tank01.Level.HiHi", target.AlarmFullReference); } + /// + /// GWC-37: swallowing the discovery failure is right for the resolver but leaves the + /// caller unable to tell "discovery produced nothing" from "nothing was configured", which + /// is why the empty watch-list could sit unnoticed for a monitor's whole life. The + /// resolution therefore reports whether discovery was attempted, whether it failed, and + /// the failure message the provider status quotes. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task ResolveAsync_RepositoryThrows_ReportsDiscoveryFailureOnTheResolution() + { + ThrowingGalaxyRepository repo = new(new InvalidOperationException("SQL unavailable")); + + AlarmWatchListResolver resolver = CreateResolver(repo); + + AlarmWatchListResolution resolution = await resolver.ResolveAsync(Options()); + + Assert.Empty(resolution.Targets); + Assert.True(resolution.DiscoveryAttempted); + Assert.True(resolution.DiscoveryFailed); + Assert.Equal("SQL unavailable", resolution.DiscoveryFailureMessage); + } + + /// + /// The complement of the case above: discovery disabled is not discovery failed. An empty + /// watch-list here means the deployment did not ask for subtag coverage, which the monitor + /// must not report as a degraded standby (GWC-37). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task ResolveAsync_RepositoryDisabled_ReportsDiscoveryNotAttempted() + { + StubGalaxyRepository repo = new([]); + + AlarmWatchListResolver resolver = CreateResolver(repo); + + AlarmWatchListResolution resolution = await resolver.ResolveAsync( + Options(useGalaxyRepository: false)); + + Assert.Empty(resolution.Targets); + Assert.False(resolution.DiscoveryAttempted); + Assert.False(resolution.DiscoveryFailed); + Assert.Null(resolution.DiscoveryFailureMessage); + } + /// Verifies the source object reference is derived from a config include entry, using the portion before the first dot when present. /// A task that represents the asynchronous operation. [Fact] @@ -252,7 +312,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( include: ["Tank01.Level.HiHi", "StandaloneTag"])); Assert.Equal("Tank01", result[0].SourceObjectReference); @@ -273,7 +333,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( useGalaxyRepository: false, include: ["Tank01.Level.HiHi"], exclude: ["Tank01.Level.HiHi"])); @@ -299,7 +359,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( useGalaxyRepository: true, exclude: ["Tank02.Level.HiHi"])); @@ -324,7 +384,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); // The exclude array contains a whitespace-only string — should be a no-op. - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( useGalaxyRepository: true, exclude: [" "])); @@ -351,7 +411,7 @@ public sealed class AlarmWatchListResolverTests AlarmWatchListResolver resolver = CreateResolver(repo); await Assert.ThrowsAnyAsync(() => - resolver.ResolveAsync( + ResolveTargetsAsync(resolver, Options(include: ["Tank01.Level.HiHi"]), cts.Token)); } @@ -375,7 +435,7 @@ public sealed class AlarmWatchListResolverTests // Pump01.Fault is an explicit include AND an exclude (case-insensitive). It must // be dropped; the GR row Tank01 survives. - IReadOnlyList result = await resolver.ResolveAsync(Options( + IReadOnlyList result = await ResolveTargetsAsync(resolver, Options( useGalaxyRepository: true, include: ["Pump01.Fault"], exclude: ["pump01.fault"])); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs index 956cf14..4bfbb58 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs @@ -498,10 +498,14 @@ public sealed class GatewayAlarmMonitorAttachOrderTests private sealed class StubWatchListResolver : IAlarmWatchListResolver { /// - public Task> ResolveAsync( + public Task ResolveAsync( AlarmsOptions options, CancellationToken cancellationToken = default) => - Task.FromResult>([]); + Task.FromResult(new AlarmWatchListResolution( + [], + DiscoveryAttempted: false, + DiscoveryFailed: false, + DiscoveryFailureMessage: null)); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs index 3b3f8fb..06336ac 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs @@ -623,6 +623,184 @@ public sealed class GatewayAlarmMonitorProviderModeTests await monitor.StopAsync(CancellationToken.None); } + /// + /// GWC-37: a Galaxy Repository outage at startup used to leave one LogWarning and an + /// empty subtag watch-list that stayed empty for the monitor's whole life — invisible until + /// a failover found the standby covering nothing. The condition is now a standing verdict + /// on the provider status, so it reaches the alarm feed, the dashboard badge and every + /// late-joining subscriber. The mode stays ALARMMGR: the live provider is healthy, + /// it is the standby behind it that is unsound. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task SubtagCoverage_WhenDiscoveryFailsAtSubscribe_ProviderStatusIsDegradedWithReason() + { + using GatewayMetrics metrics = new(); + FakeSessionManager sessions = new(); + ScriptedWatchListResolver resolver = new(DiscoveryFailedResolution()); + using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver); + + using CancellationTokenSource cts = new(); + await monitor.StartAsync(cts.Token); + await sessions.WaitForSubscribeAsync(WaitTimeout); + + // The empty list still went to the worker — the verdict is a report, not a behaviour change. + Assert.Empty(sessions.LastSubscribeCommand!.WatchList); + + AlarmProviderStatus status = await ReadFirstProviderStatusAsync(monitor); + + Assert.Equal(AlarmProviderMode.Alarmmgr, status.Mode); + Assert.True(status.Degraded); + Assert.Contains(AlarmProviderReasons.SubtagWatchListEmpty, status.Reason, StringComparison.Ordinal); + Assert.Contains(AlarmProviderReasons.SubtagDiscoveryFailed, status.Reason, StringComparison.Ordinal); + Assert.Contains("gr offline", status.Reason, StringComparison.Ordinal); + + await cts.CancelAsync(); + await monitor.StopAsync(CancellationToken.None); + } + + /// + /// GWC-37: the failover edge is the moment the watch-list stops being theoretical, so the + /// monitor re-resolves it there and publishes the fresh verdict on the same status frame + /// the failover produces. Discovery that has recovered since SubscribeAlarms is + /// reported as drift rather than applied — the worker rejects a second SubscribeAlarms + /// while subscribed, and the only way to replace the list rebuilds the failover consumer on + /// the primary that just failed. The status names both the fresh count and the sent count so + /// an operator knows a monitor restart will pick the new list up. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task SubtagCoverage_WhenDiscoveryRecoversByFailover_ReResolvesAndReportsDrift() + { + using GatewayMetrics metrics = new(); + FakeSessionManager sessions = new(); + ScriptedWatchListResolver resolver = new( + DiscoveryFailedResolution(), + new AlarmWatchListResolution( + [Target("Galaxy!Area.Tank01.Level.HiHi"), Target("Galaxy!Area.Tank02.Level.HiHi")], + DiscoveryAttempted: true, + DiscoveryFailed: false, + DiscoveryFailureMessage: null)); + using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver); + + using CancellationTokenSource cts = new(); + await monitor.StartAsync(cts.Token); + await sessions.WaitForSubscribeAsync(WaitTimeout); + Assert.Equal(1, resolver.Calls); + + List received = []; + TaskCompletionSource baselineReceived = new(TaskCreationOptions.RunContinuationsAsynchronously); + using CancellationTokenSource streamCts = new(); + Task reader = Task.Run(async () => + { + try + { + await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, streamCts.Token)) + { + lock (received) + { + received.Add(message); + if (received.Count == 1) + { + baselineReceived.TrySetResult(); + } + } + } + } + catch (OperationCanceledException) + { + // Expected when the test cancels the stream. + } + }); + + await baselineReceived.Task.WaitAsync(WaitTimeout); + + sessions.EmitEvent(new MxEvent + { + OnAlarmProviderModeChanged = new OnAlarmProviderModeChangedEvent + { + Mode = AlarmProviderMode.Subtag, + Reason = "alarmmgr failed", + At = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + }, + }); + + AlarmFeedMessage degraded = await WaitForAsync( + received, + m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus + && m.ProviderStatus.Mode == AlarmProviderMode.Subtag, + WaitTimeout); + + // The re-resolve happened on the failover edge, before the status was published. + Assert.Equal(2, resolver.Calls); + + string reason = degraded.ProviderStatus.Reason; + Assert.Contains("alarmmgr failed", reason, StringComparison.Ordinal); + Assert.Contains(AlarmProviderReasons.SubtagWatchListStale, reason, StringComparison.Ordinal); + Assert.Contains("2 target(s) now resolve against 0 sent", reason, StringComparison.Ordinal); + + // The stale-list warning replaced the empty/failed pair: discovery works again, so the only + // remaining problem is that the worker is still using the list it was handed. + Assert.DoesNotContain(AlarmProviderReasons.SubtagWatchListEmpty, reason, StringComparison.Ordinal); + Assert.DoesNotContain(AlarmProviderReasons.SubtagDiscoveryFailed, reason, StringComparison.Ordinal); + + await streamCts.CancelAsync(); + await reader; + await cts.CancelAsync(); + await monitor.StopAsync(CancellationToken.None); + } + + /// Reads the provider-status frame every stream opens with, then closes the stream. + /// The monitor to read from. + /// The provider status carried by the first frame. + private static async Task ReadFirstProviderStatusAsync(GatewayAlarmMonitor monitor) + { + using CancellationTokenSource streamCts = new(WaitTimeout); + await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, streamCts.Token)) + { + if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus) + { + return message.ProviderStatus; + } + } + + throw new TimeoutException("No provider-status message was received in time."); + } + + /// A resolution standing for "Galaxy Repository discovery was on and it failed". + /// The resolution. + private static AlarmWatchListResolution DiscoveryFailedResolution() => new( + [], + DiscoveryAttempted: true, + DiscoveryFailed: true, + DiscoveryFailureMessage: "gr offline"); + + private static AlarmSubtagTarget Target(string fullReference) => new() + { + AlarmFullReference = fullReference, + SourceObjectReference = fullReference, + ActiveSubtag = $"{fullReference}.InAlarm", + AckedSubtag = $"{fullReference}.Acked", + }; + + private static GatewayAlarmMonitor CreateMonitor( + FakeSessionManager sessions, + GatewayMetrics metrics, + IAlarmWatchListResolver resolver) + { + AlarmsOptions options = new() + { + Enabled = true, + SubscriptionExpression = @"\\NODE\Galaxy!Area", + }; + return new GatewayAlarmMonitor( + sessions, + resolver, + metrics, + Microsoft.Extensions.Options.Options.Create(new GatewayOptions { Alarms = options }), + NullLogger.Instance); + } + private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics) { AlarmsOptions options = new() @@ -693,13 +871,42 @@ public sealed class GatewayAlarmMonitorProviderModeTests throw new TimeoutException("Condition was not met in time."); } + /// + /// that hands out a scripted sequence of + /// resolutions — one per call, the last repeating — and counts the calls, so a test can + /// drive "discovery was down at subscribe and back up by the failover" and assert that the + /// second resolution happened at all (GWC-37). + /// + /// Resolutions to return, in call order. + private sealed class ScriptedWatchListResolver(params AlarmWatchListResolution[] script) : IAlarmWatchListResolver + { + private int _calls; + + /// Gets the number of times has been called. + public int Calls => Volatile.Read(ref _calls); + + /// + public Task ResolveAsync( + AlarmsOptions options, + CancellationToken cancellationToken = default) + { + int index = Interlocked.Increment(ref _calls) - 1; + return Task.FromResult(script[Math.Min(index, script.Length - 1)]); + } + } + /// that returns a fixed watch-list. private sealed class StubWatchListResolver(IReadOnlyList targets) : IAlarmWatchListResolver { /// - public Task> ResolveAsync( + public Task ResolveAsync( AlarmsOptions options, - CancellationToken cancellationToken = default) => Task.FromResult(targets); + CancellationToken cancellationToken = default) => + Task.FromResult(new AlarmWatchListResolution( + targets, + DiscoveryAttempted: false, + DiscoveryFailed: false, + DiscoveryFailureMessage: null)); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardBrowseAndAlarmModelTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardBrowseAndAlarmModelTests.cs index a4a6de2..55ca53b 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardBrowseAndAlarmModelTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardBrowseAndAlarmModelTests.cs @@ -179,8 +179,10 @@ public sealed class DashboardBrowseAndAlarmModelTests /// /// An explicitly-degraded status whose mode is still Alarmmgr (the - /// Degraded || Mode==Subtag guard's second, independent branch) must still - /// map to the degraded amber badge. + /// Degraded || Mode==Subtag guard's second, independent branch) must map to the + /// amber badge, but not to the failover label: GWC-37 gives that combination a meaning — + /// the alarm manager is serving and the subtag standby behind it covers nothing — so + /// reporting "Subtag monitoring (degraded)" would name the wrong live provider. /// [Fact] public void FromProviderStatus_Alarmmgr_DegradedFlagSet_WarningBadge() @@ -195,8 +197,9 @@ public sealed class DashboardBrowseAndAlarmModelTests DashboardAlarmProviderStatus model = DashboardAlarmProviderStatus.FromProviderStatus(status); Assert.True(model.IsDegraded); - Assert.Equal(DashboardAlarmProviderStatus.DegradedLabel, model.Label); + Assert.Equal(DashboardAlarmProviderStatus.NoSubtagStandbyLabel, model.Label); Assert.Contains("bg-warning", model.BadgeCssClass, StringComparison.Ordinal); + Assert.Equal(AlarmProviderMode.Alarmmgr, model.Mode); } ///