Merge fix/c3-session-lifecycle: GWC-32 WorkerClient dispose is terminal, GWC-34 attach-after-dispose throws SessionNotReady, GWC-37 subtag watch-list coverage graded on provider status + re-resolved at failover edge
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
@@ -19,14 +18,18 @@ public sealed class EmptyAlarmWatchListResolver : IAlarmWatchListResolver
|
||||
/// <summary>Shared no-op instance.</summary>
|
||||
public static readonly EmptyAlarmWatchListResolver Instance = new();
|
||||
|
||||
private static readonly IReadOnlyList<AlarmSubtagTarget> Empty = [];
|
||||
private static readonly AlarmWatchListResolution Empty = new(
|
||||
[],
|
||||
DiscoveryAttempted: false,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null);
|
||||
|
||||
private EmptyAlarmWatchListResolver()
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
|
||||
@@ -15,4 +15,32 @@ public static class AlarmProviderReasons
|
||||
/// distinguish a forced subtag mode from an unexpected degraded failover.
|
||||
/// </summary>
|
||||
public const string ForcedSubtag = "Forced subtag mode (configuration)";
|
||||
|
||||
/// <summary>
|
||||
/// Reason prefix set when the effective mode can need the subtag watch-list
|
||||
/// (<c>Auto</c> or <c>ForceSubtag</c>) but the list resolved to zero targets, so the
|
||||
/// subtag provider covers no alarms (GWC-37). In <c>Auto</c> this is a warning about the
|
||||
/// standby, not about the currently-serving alarm manager, which is why the dashboard
|
||||
/// renders it against the <c>ALARMMGR</c> mode as a coverage badge rather than a failover.
|
||||
/// </summary>
|
||||
public const string SubtagWatchListEmpty =
|
||||
"Subtag watch-list is empty: the subtag provider covers no alarms";
|
||||
|
||||
/// <summary>
|
||||
/// Reason prefix set when Galaxy Repository alarm discovery was enabled but failed, so
|
||||
/// the watch-list carries only what configuration contributed and may be incomplete
|
||||
/// (GWC-37). Distinct from <see cref="SubtagWatchListEmpty"/>: discovery failing with a
|
||||
/// usable config-only list is a completeness warning, not a total loss of coverage.
|
||||
/// </summary>
|
||||
public const string SubtagDiscoveryFailed =
|
||||
"Galaxy Repository alarm discovery failed: the subtag watch-list may be incomplete";
|
||||
|
||||
/// <summary>
|
||||
/// Reason prefix set when a re-resolution at the failover edge produced a different
|
||||
/// watch-list than the one sent at <c>SubscribeAlarms</c> (GWC-37). The worker keeps the
|
||||
/// list it was given for the life of the subscription, so the difference is reported
|
||||
/// rather than applied; a monitor restart re-resolves and re-sends.
|
||||
/// </summary>
|
||||
public const string SubtagWatchListStale =
|
||||
"Subtag watch-list differs from the one the worker is using";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of one subtag watch-list resolution: the composed targets plus enough
|
||||
/// provenance for the caller to tell an empty list that means "nothing was configured"
|
||||
/// from one that means "discovery was configured and silently produced nothing"
|
||||
/// (GWC-37). <see cref="IAlarmWatchListResolver.ResolveAsync"/> deliberately never throws
|
||||
/// on a discovery failure, so without this the two are indistinguishable at the call site
|
||||
/// and a Galaxy Repository outage at startup can only be found in a log line.
|
||||
/// </summary>
|
||||
/// <param name="Targets">The resolved watch-list, in order; possibly empty.</param>
|
||||
/// <param name="DiscoveryAttempted">
|
||||
/// Galaxy Repository discovery was enabled for this resolution
|
||||
/// (<c>Fallback:Discovery:UseGalaxyRepository</c>).
|
||||
/// </param>
|
||||
/// <param name="DiscoveryFailed">
|
||||
/// Discovery was attempted and failed, so <paramref name="Targets"/> holds only what
|
||||
/// configuration contributed.
|
||||
/// </param>
|
||||
/// <param name="DiscoveryFailureMessage">
|
||||
/// The discovery failure's message, or <c>null</c> when discovery did not fail.
|
||||
/// </param>
|
||||
public sealed record AlarmWatchListResolution(
|
||||
IReadOnlyList<AlarmSubtagTarget> Targets,
|
||||
bool DiscoveryAttempted,
|
||||
bool DiscoveryFailed,
|
||||
string? DiscoveryFailureMessage);
|
||||
@@ -31,7 +31,7 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public async Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -39,6 +39,11 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
|
||||
AlarmDiscoveryOptions discovery = options.Fallback.Discovery;
|
||||
|
||||
// GWC-37: the caller cannot grade an empty watch-list without knowing whether discovery
|
||||
// was asked to contribute and whether it failed while doing so, because the failure is
|
||||
// deliberately swallowed below.
|
||||
string? discoveryFailureMessage = null;
|
||||
|
||||
// Config fallback area used only for explicit IncludeAttributes entries (which
|
||||
// carry no discovered area): discovery area, else the default area (may be empty).
|
||||
string configFallbackArea = string.IsNullOrEmpty(discovery.Area) ? options.DefaultArea : discovery.Area;
|
||||
@@ -68,6 +73,7 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Galaxy Repository alarm-attribute discovery failed; continuing with configuration-only watch-list.");
|
||||
discoveryFailureMessage = ex.Message;
|
||||
rows = [];
|
||||
}
|
||||
|
||||
@@ -148,7 +154,11 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
_logger.LogDebug("Resolved alarm subtag watch-list with {TargetCount} target(s).", targets.Count);
|
||||
}
|
||||
|
||||
return targets;
|
||||
return new AlarmWatchListResolution(
|
||||
targets,
|
||||
DiscoveryAttempted: discovery.UseGalaxyRepository,
|
||||
DiscoveryFailed: discoveryFailureMessage is not null,
|
||||
discoveryFailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -57,6 +57,20 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
private string _providerReason = string.Empty;
|
||||
private DateTimeOffset _providerSince = DateTimeOffset.UtcNow;
|
||||
|
||||
// GWC-37 subtag-coverage verdict, guarded by _sync: whether the watch-list the worker is
|
||||
// using can actually cover anything, and why not. Independent of _providerDegraded (which
|
||||
// means "the subtag provider is the one serving"): in Auto mode the alarm manager can be
|
||||
// serving perfectly while the standby behind it covers zero alarms, and that is invisible
|
||||
// until the failover that needs it. Folded into BuildProviderStatus so it reaches the feed,
|
||||
// the dashboard badge and every late-joining subscriber rather than only a startup log line.
|
||||
private bool _subtagCoverageMissing;
|
||||
private string _subtagCoverageReason = string.Empty;
|
||||
|
||||
// The watch-list references actually sent to the worker in SubscribeAlarms, guarded by _sync.
|
||||
// The worker holds that list for the life of the subscription, so a later re-resolution is
|
||||
// compared against this to report drift rather than silently diverging.
|
||||
private IReadOnlyCollection<string> _sentWatchListReferences = [];
|
||||
|
||||
// Whether the worker's most recent reconcile fetch was capped, guarded by _sync.
|
||||
// Written only by ApplyReconcile (and cleared with the cache), so it describes the last full
|
||||
// reconcile — not necessarily the current _alarms contents, which live transitions keep moving
|
||||
@@ -329,12 +343,25 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
|
||||
private async Task SubscribeAlarmsAsync(string sessionId, string subscription, CancellationToken cancellationToken)
|
||||
{
|
||||
IReadOnlyList<AlarmSubtagTarget> watchList = await _watchListResolver
|
||||
AlarmWatchListResolution resolution = await _watchListResolver
|
||||
.ResolveAsync(_options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
IReadOnlyList<AlarmSubtagTarget> watchList = resolution.Targets;
|
||||
|
||||
AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode);
|
||||
|
||||
// GWC-37: grade the coverage before the command goes out, so a Galaxy Repository outage at
|
||||
// startup is a standing status on the feed instead of one log line an operator reads weeks
|
||||
// later — when the failover that needed the watch-list finds it empty. Broadcast because
|
||||
// feed subscribers outlive the monitor's worker session and may already be attached.
|
||||
lock (_sync)
|
||||
{
|
||||
_sentWatchListReferences = watchList
|
||||
.Select(target => target.AlarmFullReference)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false), broadcast: true);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Alarm subscribe: forcedMode={ForcedMode} configMode={ConfigMode} watchList={WatchListCount}.",
|
||||
forcedMode, _options.Fallback.Mode, watchList.Count);
|
||||
@@ -518,6 +545,20 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
AlarmProviderMode toMode = change.Mode;
|
||||
string reason = change.Reason ?? string.Empty;
|
||||
|
||||
// GWC-37: this is the moment the subtag watch-list stops being theoretical — the provider
|
||||
// that is about to serve is the one built from it. Re-resolve BEFORE the status is
|
||||
// published so the frame the operator sees on the failover carries the coverage verdict
|
||||
// rather than a second frame later, and so a watch-list that has drifted since
|
||||
// SubscribeAlarms is reported instead of silently diverging. The re-resolution is NOT
|
||||
// re-sent: the worker rejects a second SubscribeAlarms while subscribed, and the only way
|
||||
// to replace the list — Unsubscribe then Subscribe — rebuilds the FailoverAlarmConsumer,
|
||||
// which restarts on the primary that just failed and discards the warm standby state, so
|
||||
// it would flap the feed rather than repair it. A monitor restart re-resolves and re-sends.
|
||||
if (toMode == AlarmProviderMode.Subtag)
|
||||
{
|
||||
await RefreshSubtagCoverageAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
AlarmProviderStatus status;
|
||||
int fromModeInt;
|
||||
lock (_sync)
|
||||
@@ -565,14 +606,141 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
}
|
||||
}
|
||||
|
||||
// GWC-37: re-resolves the subtag watch-list and re-grades the coverage verdict without
|
||||
// touching the worker's subscription. Best-effort: a resolver that throws (only cancellation
|
||||
// and programming errors do — discovery failure is reported in the resolution) leaves the
|
||||
// verdict from the last subscribe in place, because a stale verdict is better than none.
|
||||
// Does not broadcast: the callers publish a status frame of their own immediately after.
|
||||
private async Task RefreshSubtagCoverageAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode);
|
||||
AlarmWatchListResolution resolution;
|
||||
try
|
||||
{
|
||||
resolution = await _watchListResolver.ResolveAsync(_options, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Re-resolving the alarm subtag watch-list failed; keeping the coverage verdict from the last subscribe.");
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_sync)
|
||||
{
|
||||
bool stale = !resolution.Targets
|
||||
.Select(target => target.AlarmFullReference)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase)
|
||||
.SetEquals(_sentWatchListReferences);
|
||||
SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, stale), broadcast: false);
|
||||
}
|
||||
}
|
||||
|
||||
// Grades one resolution against the effective mode. Returns the reason the subtag coverage is
|
||||
// unsound, or an empty string when it is sound (or irrelevant, as under ForceAlarmManager).
|
||||
// Empty targets are the total-loss case; a failed discovery with a usable config-only list is
|
||||
// a completeness warning; drift is reported separately because the worker cannot be updated.
|
||||
private string GradeSubtagCoverage(
|
||||
AlarmProviderMode forcedMode,
|
||||
AlarmWatchListResolution resolution,
|
||||
bool staleAgainstSent)
|
||||
{
|
||||
// ForceAlarmManager never builds a subtag consumer, so its watch-list is genuinely unused.
|
||||
if (forcedMode == AlarmProviderMode.Alarmmgr)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Nothing was asked of discovery and nothing was configured: subtag fallback is simply not
|
||||
// in use in this deployment (Auto with an empty list subscribes alarmmgr-only), so there is
|
||||
// no coverage to warn about.
|
||||
if (resolution.Targets.Count == 0
|
||||
&& !resolution.DiscoveryAttempted
|
||||
&& _options.Fallback.Discovery.IncludeAttributes.Length == 0
|
||||
&& forcedMode != AlarmProviderMode.Subtag)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
List<string> reasons = [];
|
||||
if (resolution.Targets.Count == 0)
|
||||
{
|
||||
reasons.Add(AlarmProviderReasons.SubtagWatchListEmpty);
|
||||
}
|
||||
|
||||
if (resolution.DiscoveryFailed)
|
||||
{
|
||||
reasons.Add(resolution.DiscoveryFailureMessage is { Length: > 0 } message
|
||||
? $"{AlarmProviderReasons.SubtagDiscoveryFailed} ({message})"
|
||||
: AlarmProviderReasons.SubtagDiscoveryFailed);
|
||||
}
|
||||
|
||||
if (staleAgainstSent)
|
||||
{
|
||||
reasons.Add(
|
||||
$"{AlarmProviderReasons.SubtagWatchListStale} ({resolution.Targets.Count} target(s) now "
|
||||
+ $"resolve against {_sentWatchListReferences.Count} sent at subscribe); restart the alarm "
|
||||
+ "monitor to re-send it");
|
||||
}
|
||||
|
||||
return string.Join("; ", reasons);
|
||||
}
|
||||
|
||||
// Caller holds _sync. Applies a coverage verdict, logging and (optionally) broadcasting only
|
||||
// when it actually changed so a stable degraded state does not repeat a frame every pass.
|
||||
private void SetSubtagCoverage(string reason, bool broadcast)
|
||||
{
|
||||
bool missing = reason.Length > 0;
|
||||
if (missing == _subtagCoverageMissing && string.Equals(reason, _subtagCoverageReason, StringComparison.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_subtagCoverageMissing = missing;
|
||||
_subtagCoverageReason = reason;
|
||||
|
||||
if (missing)
|
||||
{
|
||||
_logger.LogWarning("Alarm subtag coverage is unsound: {Reason}", reason);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Alarm subtag coverage is sound.");
|
||||
}
|
||||
|
||||
if (broadcast)
|
||||
{
|
||||
BroadcastToAll(new AlarmFeedMessage { ProviderStatus = BuildProviderStatus() });
|
||||
}
|
||||
}
|
||||
|
||||
// Caller holds _sync. Builds an AlarmProviderStatus snapshot of the current state.
|
||||
//
|
||||
// GWC-37: degraded is the OR of "the subtag provider is the one serving" and "the subtag
|
||||
// watch-list cannot cover anything". The second is what makes an empty watch-list visible in
|
||||
// Auto mode, where the alarm manager is serving happily and the standby behind it is empty.
|
||||
// The reasons are concatenated when both apply, which is the worst case worth naming loudly:
|
||||
// failed over to a provider that covers nothing.
|
||||
private AlarmProviderStatus BuildProviderStatus()
|
||||
{
|
||||
bool degraded = _providerDegraded || _subtagCoverageMissing;
|
||||
string reason = (_providerReason, _subtagCoverageReason) switch
|
||||
{
|
||||
("", var coverage) => coverage,
|
||||
(var mode, "") => mode,
|
||||
(var mode, var coverage) => $"{mode} — {coverage}",
|
||||
};
|
||||
|
||||
return new AlarmProviderStatus
|
||||
{
|
||||
Mode = _providerMode,
|
||||
Degraded = _providerDegraded,
|
||||
Reason = _providerReason,
|
||||
Degraded = degraded,
|
||||
Reason = reason,
|
||||
Since = Timestamp.FromDateTimeOffset(_providerSince),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,13 +18,15 @@ public interface IAlarmWatchListResolver
|
||||
/// <param name="options">Alarm configuration carrying discovery and subtag-name settings.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>
|
||||
/// The resolved <see cref="AlarmSubtagTarget"/> watch-list, possibly empty.
|
||||
/// The resolved <see cref="AlarmSubtagTarget"/> watch-list, possibly empty, together
|
||||
/// with the discovery provenance the caller needs to grade it.
|
||||
/// Discovery being unavailable never throws — it yields an empty (or
|
||||
/// config-only) list and the caller decides what to do with it. Cancellation
|
||||
/// config-only) list with <see cref="AlarmWatchListResolution.DiscoveryFailed"/> set,
|
||||
/// and the caller decides what to do with it. Cancellation
|
||||
/// is the one exception: a triggered <paramref name="cancellationToken"/>
|
||||
/// still propagates an <see cref="OperationCanceledException"/>.
|
||||
/// </returns>
|
||||
Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -30,6 +30,14 @@ public sealed record DashboardAlarmProviderStatus(
|
||||
/// </summary>
|
||||
public const string ForcedSubtagLabel = "Subtag monitoring (forced)";
|
||||
|
||||
/// <summary>
|
||||
/// Badge label shown when the alarm manager is serving normally but the subtag standby
|
||||
/// behind it covers nothing — an empty or undiscoverable watch-list (GWC-37). The feed is
|
||||
/// healthy today; what is degraded is the fallback that would take over tomorrow, so the
|
||||
/// badge names the standby rather than claiming the feed has failed over.
|
||||
/// </summary>
|
||||
public const string NoSubtagStandbyLabel = "Alarm Manager (no subtag standby)";
|
||||
|
||||
private const string HealthyBadge = "bg-success";
|
||||
private const string DegradedBadge = "bg-warning text-dark";
|
||||
|
||||
@@ -69,8 +77,18 @@ public sealed record DashboardAlarmProviderStatus(
|
||||
&& status.Mode == AlarmProviderMode.Subtag
|
||||
&& string.Equals(reason, AlarmProviderReasons.ForcedSubtag, StringComparison.Ordinal);
|
||||
|
||||
string label = !degraded ? AlarmManagerLabel : forced ? ForcedSubtagLabel : DegradedLabel;
|
||||
string badge = !degraded ? HealthyBadge : forced ? ForcedSubtagBadge : DegradedBadge;
|
||||
// GWC-37: degraded WITHOUT subtag mode means the alarm manager is still serving and the
|
||||
// standby is the unsound half. "Subtag monitoring (degraded)" would misreport which
|
||||
// provider is live, so this state gets its own label; the amber badge is shared because
|
||||
// the operator action — fix discovery — is equally overdue in both.
|
||||
bool standbyOnlyWarning = degraded && status.Mode != AlarmProviderMode.Subtag;
|
||||
|
||||
string label = !degraded
|
||||
? AlarmManagerLabel
|
||||
: standbyOnlyWarning ? NoSubtagStandbyLabel : forced ? ForcedSubtagLabel : DegradedLabel;
|
||||
string badge = !degraded
|
||||
? HealthyBadge
|
||||
: standbyOnlyWarning ? DegradedBadge : forced ? ForcedSubtagBadge : DegradedBadge;
|
||||
|
||||
return new DashboardAlarmProviderStatus(
|
||||
Mode: status.Mode,
|
||||
|
||||
@@ -39,6 +39,14 @@ public sealed class GatewaySession
|
||||
private bool _everHadEventSubscriber;
|
||||
private SessionEventDistributor? _eventDistributor;
|
||||
private bool _eventDistributorStarted;
|
||||
// GWC-34: set under _syncRoot when DisposeAsync nulls _eventDistributor, making the
|
||||
// "one distributor per session, never reset" lifecycle symmetric at both ends. Without it
|
||||
// EnsureDistributorCreated reads the null as "not created yet" and builds a REPLACEMENT
|
||||
// distributor after teardown — one whose pump can never start (_eventDistributorStarted is a
|
||||
// one-way latch) and whose channel is never completed, so an attach that stalled across the
|
||||
// close+dispose registers on it and blocks in WaitToReadAsync until the client's own deadline,
|
||||
// silently and invisibly to metrics.
|
||||
private bool _eventDistributorDisposed;
|
||||
private bool _dashboardMirrorStarted;
|
||||
private IEventSubscriberLease? _dashboardMirrorLease;
|
||||
private Task? _dashboardMirrorTask;
|
||||
@@ -532,10 +540,23 @@ public sealed class GatewaySession
|
||||
// start). Both the construction and the started-flag flip happen under _syncRoot so two
|
||||
// concurrent callers (e.g. MarkReady's dashboard mirror and a racing first
|
||||
// AttachEventSubscriber) agree on a single distributor and a single start.
|
||||
//
|
||||
// GWC-34: after DisposeAsync the null distributor means "gone", not "not built yet", so this
|
||||
// throws instead of constructing a replacement nothing will ever pump. The attach paths already
|
||||
// roll back and surface the exception, which is the same SessionNotReady the readiness gates
|
||||
// raise — a caller that loses the race to teardown now fails immediately instead of hanging.
|
||||
private SessionEventDistributor EnsureDistributorCreated(out bool startNow)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_eventDistributorDisposed)
|
||||
{
|
||||
throw new SessionManagerException(
|
||||
SessionManagerErrorCode.SessionNotReady,
|
||||
$"Session {SessionId} is not ready for event streaming. Its event distributor was "
|
||||
+ "disposed with the session.");
|
||||
}
|
||||
|
||||
if (_eventDistributor is null)
|
||||
{
|
||||
EventOptions eventOptions = _eventStreaming.EventOptions;
|
||||
@@ -672,7 +693,37 @@ public sealed class GatewaySession
|
||||
// dashboard mirror only disconnects itself and never faults the session.
|
||||
// These three calls are OUTSIDE _syncRoot to avoid holding it across
|
||||
// EnsureDistributorCreated's own lock and StartAsync's Task.Run.
|
||||
SessionEventDistributor distributor = EnsureDistributorCreated(out bool startNow);
|
||||
//
|
||||
// GWC-34: the state check above can be overtaken by a close+dispose before we get here, and
|
||||
// EnsureDistributorCreated now says so instead of building an orphan distributor. The
|
||||
// dashboard mirror is a best-effort observability attachment on a session that is already
|
||||
// gone, so it is abandoned here rather than propagated out of MarkReady — the mirror-started
|
||||
// latch stays set (nothing may start one on a disposed session) and the loop's token source
|
||||
// is released so it is not orphaned.
|
||||
SessionEventDistributor distributor;
|
||||
bool startNow;
|
||||
try
|
||||
{
|
||||
distributor = EnsureDistributorCreated(out startNow);
|
||||
}
|
||||
catch (SessionManagerException exception)
|
||||
{
|
||||
CancellationTokenSource? abandonedCts;
|
||||
lock (_syncRoot)
|
||||
{
|
||||
abandonedCts = _dashboardMirrorCts;
|
||||
_dashboardMirrorCts = null;
|
||||
}
|
||||
|
||||
abandonedCts?.Dispose();
|
||||
_eventStreaming.DistributorLogger.LogDebug(
|
||||
exception,
|
||||
"Dashboard event mirror for session {SessionId} was abandoned: the session was disposed "
|
||||
+ "before the mirror could attach.",
|
||||
SessionId);
|
||||
return;
|
||||
}
|
||||
|
||||
IEventSubscriberLease lease = distributor.Register(isInternal: true);
|
||||
StartPumpIfRequested(distributor, startNow);
|
||||
|
||||
@@ -1867,6 +1918,9 @@ public sealed class GatewaySession
|
||||
{
|
||||
distributor = _eventDistributor;
|
||||
_eventDistributor = null;
|
||||
// GWC-34: latch the null-out so EnsureDistributorCreated cannot read it as
|
||||
// "not created yet" and hand a racing attach a distributor that will never pump.
|
||||
_eventDistributorDisposed = true;
|
||||
}
|
||||
|
||||
if (distributor is not null)
|
||||
|
||||
@@ -420,12 +420,32 @@ public sealed class WorkerClient : IWorkerClient
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
// GWC-32: disposal is the third channel-completing path, and it must enter a terminal state
|
||||
// like the other two (MarkClosed, SetFaulted) BEFORE the channels are completed. Otherwise a
|
||||
// client disposed while still Ready keeps reporting IsTerminalState() == false, and a read
|
||||
// loop already inside DispatchEnvelope (the _stopCts.Cancel below only interrupts the NEXT
|
||||
// ReadAsync) takes StageWorkerEvent's overflow arm on the completed staging channel: a
|
||||
// spurious ProtocolViolation fault, a worker-event-staging overflow metric and a redundant
|
||||
// kill, all during an intentional teardown. MarkClosed is deliberately not reused here: it
|
||||
// would overwrite Faulted with Closed and it re-runs the whole teardown sequence this method
|
||||
// performs itself. The transition is skipped when the state is already terminal, so a
|
||||
// dispose-after-fault keeps Faulted and a double dispose changes nothing.
|
||||
bool becameTerminalHere;
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return;
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
becameTerminalHere = _state is not (WorkerClientState.Closed or WorkerClientState.Faulted);
|
||||
if (becameTerminalHere)
|
||||
{
|
||||
_state = WorkerClientState.Closed;
|
||||
}
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
// Drop out of the worker queue-depth gauge before teardown: whatever this client still holds
|
||||
// is about to be discarded, and the sum must not keep counting a client that is going away.
|
||||
_eventQueueDepthRegistration?.Dispose();
|
||||
@@ -434,6 +454,16 @@ public sealed class WorkerClient : IWorkerClient
|
||||
_outboundEnvelopes.Writer.TryComplete();
|
||||
_eventStaging.Writer.TryComplete();
|
||||
_events.Writer.TryComplete();
|
||||
if (becameTerminalHere)
|
||||
{
|
||||
// A worker that reached Ready and is now being disposed without a prior Close or fault
|
||||
// (SessionManager.OpenSessionAsync's failure arm does exactly this) stopped, so the
|
||||
// running-worker gauge must come down. RecordWorkerStoppedOnce is latched on
|
||||
// _workerStartRecorded, so a client that already recorded its stop through MarkClosed or
|
||||
// SetFaulted cannot record a second one here.
|
||||
RecordWorkerStoppedOnce("dispose");
|
||||
}
|
||||
|
||||
CompletePendingCommands(
|
||||
new WorkerClientException(
|
||||
WorkerClientErrorCode.GatewayShutdown,
|
||||
@@ -647,8 +677,10 @@ public sealed class WorkerClient : IWorkerClient
|
||||
/// Hands a received worker event to the dedicated event writer without blocking the read loop.
|
||||
/// <c>TryWrite</c> is non-blocking, so the read loop keeps dispatching replies, heartbeats and
|
||||
/// faults regardless of how backed up the event path is. It returns <c>false</c> in two cases:
|
||||
/// the staging channel has been completed during shutdown (the event is safely dropped because
|
||||
/// the client is already terminal), or the channel is full at its
|
||||
/// the staging channel has been completed by one of the three completing paths —
|
||||
/// <see cref="MarkClosed"/>, <see cref="SetFaulted"/> or <see cref="DisposeAsync"/>, each of
|
||||
/// which enters a terminal state before completing the channels (GWC-32), so the event is
|
||||
/// safely dropped because the client is already terminal — or the channel is full at its
|
||||
/// 2 × <see cref="WorkerClientOptions.EventChannelCapacity"/> bound. The latter means the
|
||||
/// consumer has been draining slower than the worker produces for the whole time it took the
|
||||
/// worker to emit that many further events, so the client is faulted immediately (GWC-24) —
|
||||
|
||||
@@ -384,9 +384,14 @@ public sealed class AlarmFailoverEndToEndTests
|
||||
private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default) => Task.FromResult(targets);
|
||||
CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(new AlarmWatchListResolution(
|
||||
targets,
|
||||
DiscoveryAttempted: false,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -507,10 +507,14 @@ public sealed class AlarmTruncationSignalTests
|
||||
private sealed class StubWatchListResolver : IAlarmWatchListResolver
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<IReadOnlyList<AlarmSubtagTarget>>([]);
|
||||
Task.FromResult(new AlarmWatchListResolution(
|
||||
[],
|
||||
DiscoveryAttempted: false,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -16,6 +16,21 @@ public sealed class AlarmWatchListResolverTests
|
||||
private static AlarmWatchListResolver CreateResolver(IGalaxyRepository repository) =>
|
||||
new(repository, NullLogger<AlarmWatchListResolver>.Instance);
|
||||
|
||||
/// <summary>
|
||||
/// Resolves and projects to the target list. These cases assert composition rules, which
|
||||
/// live in <see cref="AlarmWatchListResolution.Targets"/>; the discovery-provenance fields
|
||||
/// the resolution also carries (GWC-37) are asserted by their own cases below.
|
||||
/// </summary>
|
||||
/// <param name="resolver">The resolver under test.</param>
|
||||
/// <param name="options">Alarm options to resolve against.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the resolution.</param>
|
||||
/// <returns>The resolved targets.</returns>
|
||||
private static async Task<IReadOnlyList<AlarmSubtagTarget>> ResolveTargetsAsync(
|
||||
AlarmWatchListResolver resolver,
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
(await resolver.ResolveAsync(options, cancellationToken)).Targets;
|
||||
|
||||
private static AlarmsOptions Options(
|
||||
bool useGalaxyRepository = true,
|
||||
string area = "",
|
||||
@@ -58,7 +73,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
include: ["pump01.fault", "Valve03.Position.Lo"],
|
||||
exclude: ["Tank02.Level.HiHi"]));
|
||||
|
||||
@@ -81,7 +96,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
subtags: new AlarmSubtagNameOptions
|
||||
{
|
||||
Active = "InAlarm",
|
||||
@@ -110,7 +125,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
// Default Priority is "Priority"; force it empty alongside empty AckComment.
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
subtags: new AlarmSubtagNameOptions { Priority = string.Empty, AckComment = string.Empty }));
|
||||
|
||||
AlarmSubtagTarget target = Assert.Single(result);
|
||||
@@ -140,7 +155,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
// Config area "DEV" must be ignored for a GR row that has a discovered area.
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver,
|
||||
Options(area: "DEV", defaultArea: "DEV"));
|
||||
|
||||
AlarmSubtagTarget target = Assert.Single(result);
|
||||
@@ -161,7 +176,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
// No discovery area and no default area.
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options());
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options());
|
||||
|
||||
AlarmSubtagTarget target = Assert.Single(result);
|
||||
Assert.Equal("Galaxy!Tank01.Level.HiHi", target.AlarmFullReference);
|
||||
@@ -178,7 +193,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
area: "Site_A",
|
||||
include: ["Tank01.Level.HiHi"]));
|
||||
|
||||
@@ -196,7 +211,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
area: "",
|
||||
defaultArea: "Plant",
|
||||
include: ["Tank01.Level.HiHi"]));
|
||||
@@ -217,7 +232,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
useGalaxyRepository: false,
|
||||
include: ["Tank01.Level.HiHi"]));
|
||||
|
||||
@@ -235,7 +250,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
include: ["Tank01.Level.HiHi"]));
|
||||
|
||||
// Did not throw; discovery set empty, include retained.
|
||||
@@ -243,6 +258,51 @@ public sealed class AlarmWatchListResolverTests
|
||||
Assert.Equal("Galaxy!Tank01.Level.HiHi", target.AlarmFullReference);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-37: swallowing the discovery failure is right for the resolver but leaves the
|
||||
/// caller unable to tell "discovery produced nothing" from "nothing was configured", which
|
||||
/// is why the empty watch-list could sit unnoticed for a monitor's whole life. The
|
||||
/// resolution therefore reports whether discovery was attempted, whether it failed, and
|
||||
/// the failure message the provider status quotes.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ResolveAsync_RepositoryThrows_ReportsDiscoveryFailureOnTheResolution()
|
||||
{
|
||||
ThrowingGalaxyRepository repo = new(new InvalidOperationException("SQL unavailable"));
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
AlarmWatchListResolution resolution = await resolver.ResolveAsync(Options());
|
||||
|
||||
Assert.Empty(resolution.Targets);
|
||||
Assert.True(resolution.DiscoveryAttempted);
|
||||
Assert.True(resolution.DiscoveryFailed);
|
||||
Assert.Equal("SQL unavailable", resolution.DiscoveryFailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The complement of the case above: discovery disabled is not discovery failed. An empty
|
||||
/// watch-list here means the deployment did not ask for subtag coverage, which the monitor
|
||||
/// must not report as a degraded standby (GWC-37).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ResolveAsync_RepositoryDisabled_ReportsDiscoveryNotAttempted()
|
||||
{
|
||||
StubGalaxyRepository repo = new([]);
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
AlarmWatchListResolution resolution = await resolver.ResolveAsync(
|
||||
Options(useGalaxyRepository: false));
|
||||
|
||||
Assert.Empty(resolution.Targets);
|
||||
Assert.False(resolution.DiscoveryAttempted);
|
||||
Assert.False(resolution.DiscoveryFailed);
|
||||
Assert.Null(resolution.DiscoveryFailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>Verifies the source object reference is derived from a config include entry, using the portion before the first dot when present.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -252,7 +312,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
include: ["Tank01.Level.HiHi", "StandaloneTag"]));
|
||||
|
||||
Assert.Equal("Tank01", result[0].SourceObjectReference);
|
||||
@@ -273,7 +333,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
useGalaxyRepository: false,
|
||||
include: ["Tank01.Level.HiHi"],
|
||||
exclude: ["Tank01.Level.HiHi"]));
|
||||
@@ -299,7 +359,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
useGalaxyRepository: true,
|
||||
exclude: ["Tank02.Level.HiHi"]));
|
||||
|
||||
@@ -324,7 +384,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
// The exclude array contains a whitespace-only string — should be a no-op.
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
useGalaxyRepository: true,
|
||||
exclude: [" "]));
|
||||
|
||||
@@ -351,7 +411,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
AlarmWatchListResolver resolver = CreateResolver(repo);
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
|
||||
resolver.ResolveAsync(
|
||||
ResolveTargetsAsync(resolver,
|
||||
Options(include: ["Tank01.Level.HiHi"]),
|
||||
cts.Token));
|
||||
}
|
||||
@@ -375,7 +435,7 @@ public sealed class AlarmWatchListResolverTests
|
||||
|
||||
// Pump01.Fault is an explicit include AND an exclude (case-insensitive). It must
|
||||
// be dropped; the GR row Tank01 survives.
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await resolver.ResolveAsync(Options(
|
||||
IReadOnlyList<AlarmSubtagTarget> result = await ResolveTargetsAsync(resolver, Options(
|
||||
useGalaxyRepository: true,
|
||||
include: ["Pump01.Fault"],
|
||||
exclude: ["pump01.fault"]));
|
||||
|
||||
@@ -498,10 +498,14 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
|
||||
private sealed class StubWatchListResolver : IAlarmWatchListResolver
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<IReadOnlyList<AlarmSubtagTarget>>([]);
|
||||
Task.FromResult(new AlarmWatchListResolution(
|
||||
[],
|
||||
DiscoveryAttempted: false,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -623,6 +623,184 @@ public sealed class GatewayAlarmMonitorProviderModeTests
|
||||
await monitor.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-37: a Galaxy Repository outage at startup used to leave one <c>LogWarning</c> and an
|
||||
/// empty subtag watch-list that stayed empty for the monitor's whole life — invisible until
|
||||
/// a failover found the standby covering nothing. The condition is now a standing verdict
|
||||
/// on the provider status, so it reaches the alarm feed, the dashboard badge and every
|
||||
/// late-joining subscriber. The mode stays <c>ALARMMGR</c>: the live provider is healthy,
|
||||
/// it is the standby behind it that is unsound.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SubtagCoverage_WhenDiscoveryFailsAtSubscribe_ProviderStatusIsDegradedWithReason()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
FakeSessionManager sessions = new();
|
||||
ScriptedWatchListResolver resolver = new(DiscoveryFailedResolution());
|
||||
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await monitor.StartAsync(cts.Token);
|
||||
await sessions.WaitForSubscribeAsync(WaitTimeout);
|
||||
|
||||
// The empty list still went to the worker — the verdict is a report, not a behaviour change.
|
||||
Assert.Empty(sessions.LastSubscribeCommand!.WatchList);
|
||||
|
||||
AlarmProviderStatus status = await ReadFirstProviderStatusAsync(monitor);
|
||||
|
||||
Assert.Equal(AlarmProviderMode.Alarmmgr, status.Mode);
|
||||
Assert.True(status.Degraded);
|
||||
Assert.Contains(AlarmProviderReasons.SubtagWatchListEmpty, status.Reason, StringComparison.Ordinal);
|
||||
Assert.Contains(AlarmProviderReasons.SubtagDiscoveryFailed, status.Reason, StringComparison.Ordinal);
|
||||
Assert.Contains("gr offline", status.Reason, StringComparison.Ordinal);
|
||||
|
||||
await cts.CancelAsync();
|
||||
await monitor.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-37: the failover edge is the moment the watch-list stops being theoretical, so the
|
||||
/// monitor re-resolves it there and publishes the fresh verdict on the same status frame
|
||||
/// the failover produces. Discovery that has recovered since <c>SubscribeAlarms</c> is
|
||||
/// reported as drift rather than applied — the worker rejects a second <c>SubscribeAlarms</c>
|
||||
/// while subscribed, and the only way to replace the list rebuilds the failover consumer on
|
||||
/// the primary that just failed. The status names both the fresh count and the sent count so
|
||||
/// an operator knows a monitor restart will pick the new list up.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SubtagCoverage_WhenDiscoveryRecoversByFailover_ReResolvesAndReportsDrift()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
FakeSessionManager sessions = new();
|
||||
ScriptedWatchListResolver resolver = new(
|
||||
DiscoveryFailedResolution(),
|
||||
new AlarmWatchListResolution(
|
||||
[Target("Galaxy!Area.Tank01.Level.HiHi"), Target("Galaxy!Area.Tank02.Level.HiHi")],
|
||||
DiscoveryAttempted: true,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null));
|
||||
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics, resolver);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await monitor.StartAsync(cts.Token);
|
||||
await sessions.WaitForSubscribeAsync(WaitTimeout);
|
||||
Assert.Equal(1, resolver.Calls);
|
||||
|
||||
List<AlarmFeedMessage> received = [];
|
||||
TaskCompletionSource baselineReceived = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using CancellationTokenSource streamCts = new();
|
||||
Task reader = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, streamCts.Token))
|
||||
{
|
||||
lock (received)
|
||||
{
|
||||
received.Add(message);
|
||||
if (received.Count == 1)
|
||||
{
|
||||
baselineReceived.TrySetResult();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected when the test cancels the stream.
|
||||
}
|
||||
});
|
||||
|
||||
await baselineReceived.Task.WaitAsync(WaitTimeout);
|
||||
|
||||
sessions.EmitEvent(new MxEvent
|
||||
{
|
||||
OnAlarmProviderModeChanged = new OnAlarmProviderModeChangedEvent
|
||||
{
|
||||
Mode = AlarmProviderMode.Subtag,
|
||||
Reason = "alarmmgr failed",
|
||||
At = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
},
|
||||
});
|
||||
|
||||
AlarmFeedMessage degraded = await WaitForAsync(
|
||||
received,
|
||||
m => m.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus
|
||||
&& m.ProviderStatus.Mode == AlarmProviderMode.Subtag,
|
||||
WaitTimeout);
|
||||
|
||||
// The re-resolve happened on the failover edge, before the status was published.
|
||||
Assert.Equal(2, resolver.Calls);
|
||||
|
||||
string reason = degraded.ProviderStatus.Reason;
|
||||
Assert.Contains("alarmmgr failed", reason, StringComparison.Ordinal);
|
||||
Assert.Contains(AlarmProviderReasons.SubtagWatchListStale, reason, StringComparison.Ordinal);
|
||||
Assert.Contains("2 target(s) now resolve against 0 sent", reason, StringComparison.Ordinal);
|
||||
|
||||
// The stale-list warning replaced the empty/failed pair: discovery works again, so the only
|
||||
// remaining problem is that the worker is still using the list it was handed.
|
||||
Assert.DoesNotContain(AlarmProviderReasons.SubtagWatchListEmpty, reason, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(AlarmProviderReasons.SubtagDiscoveryFailed, reason, StringComparison.Ordinal);
|
||||
|
||||
await streamCts.CancelAsync();
|
||||
await reader;
|
||||
await cts.CancelAsync();
|
||||
await monitor.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>Reads the provider-status frame every stream opens with, then closes the stream.</summary>
|
||||
/// <param name="monitor">The monitor to read from.</param>
|
||||
/// <returns>The provider status carried by the first frame.</returns>
|
||||
private static async Task<AlarmProviderStatus> ReadFirstProviderStatusAsync(GatewayAlarmMonitor monitor)
|
||||
{
|
||||
using CancellationTokenSource streamCts = new(WaitTimeout);
|
||||
await foreach (AlarmFeedMessage message in monitor.StreamAsync(null, streamCts.Token))
|
||||
{
|
||||
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus)
|
||||
{
|
||||
return message.ProviderStatus;
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException("No provider-status message was received in time.");
|
||||
}
|
||||
|
||||
/// <summary>A resolution standing for "Galaxy Repository discovery was on and it failed".</summary>
|
||||
/// <returns>The resolution.</returns>
|
||||
private static AlarmWatchListResolution DiscoveryFailedResolution() => new(
|
||||
[],
|
||||
DiscoveryAttempted: true,
|
||||
DiscoveryFailed: true,
|
||||
DiscoveryFailureMessage: "gr offline");
|
||||
|
||||
private static AlarmSubtagTarget Target(string fullReference) => new()
|
||||
{
|
||||
AlarmFullReference = fullReference,
|
||||
SourceObjectReference = fullReference,
|
||||
ActiveSubtag = $"{fullReference}.InAlarm",
|
||||
AckedSubtag = $"{fullReference}.Acked",
|
||||
};
|
||||
|
||||
private static GatewayAlarmMonitor CreateMonitor(
|
||||
FakeSessionManager sessions,
|
||||
GatewayMetrics metrics,
|
||||
IAlarmWatchListResolver resolver)
|
||||
{
|
||||
AlarmsOptions options = new()
|
||||
{
|
||||
Enabled = true,
|
||||
SubscriptionExpression = @"\\NODE\Galaxy!Area",
|
||||
};
|
||||
return new GatewayAlarmMonitor(
|
||||
sessions,
|
||||
resolver,
|
||||
metrics,
|
||||
Microsoft.Extensions.Options.Options.Create(new GatewayOptions { Alarms = options }),
|
||||
NullLogger<GatewayAlarmMonitor>.Instance);
|
||||
}
|
||||
|
||||
private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics)
|
||||
{
|
||||
AlarmsOptions options = new()
|
||||
@@ -693,13 +871,42 @@ public sealed class GatewayAlarmMonitorProviderModeTests
|
||||
throw new TimeoutException("Condition was not met in time.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IAlarmWatchListResolver"/> that hands out a scripted sequence of
|
||||
/// resolutions — one per call, the last repeating — and counts the calls, so a test can
|
||||
/// drive "discovery was down at subscribe and back up by the failover" and assert that the
|
||||
/// second resolution happened at all (GWC-37).
|
||||
/// </summary>
|
||||
/// <param name="script">Resolutions to return, in call order.</param>
|
||||
private sealed class ScriptedWatchListResolver(params AlarmWatchListResolution[] script) : IAlarmWatchListResolver
|
||||
{
|
||||
private int _calls;
|
||||
|
||||
/// <summary>Gets the number of times <see cref="ResolveAsync"/> has been called.</summary>
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
int index = Interlocked.Increment(ref _calls) - 1;
|
||||
return Task.FromResult(script[Math.Min(index, script.Length - 1)]);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary><see cref="IAlarmWatchListResolver"/> that returns a fixed watch-list.</summary>
|
||||
private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default) => Task.FromResult(targets);
|
||||
CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(new AlarmWatchListResolution(
|
||||
targets,
|
||||
DiscoveryAttempted: false,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+6
-3
@@ -179,8 +179,10 @@ public sealed class DashboardBrowseAndAlarmModelTests
|
||||
|
||||
/// <summary>
|
||||
/// An explicitly-degraded status whose mode is still Alarmmgr (the
|
||||
/// <c>Degraded || Mode==Subtag</c> guard's second, independent branch) must still
|
||||
/// map to the degraded amber badge.
|
||||
/// <c>Degraded || Mode==Subtag</c> guard's second, independent branch) must map to the
|
||||
/// amber badge, but not to the failover label: GWC-37 gives that combination a meaning —
|
||||
/// the alarm manager is serving and the subtag standby behind it covers nothing — so
|
||||
/// reporting "Subtag monitoring (degraded)" would name the wrong live provider.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FromProviderStatus_Alarmmgr_DegradedFlagSet_WarningBadge()
|
||||
@@ -195,8 +197,9 @@ public sealed class DashboardBrowseAndAlarmModelTests
|
||||
DashboardAlarmProviderStatus model = DashboardAlarmProviderStatus.FromProviderStatus(status);
|
||||
|
||||
Assert.True(model.IsDegraded);
|
||||
Assert.Equal(DashboardAlarmProviderStatus.DegradedLabel, model.Label);
|
||||
Assert.Equal(DashboardAlarmProviderStatus.NoSubtagStandbyLabel, model.Label);
|
||||
Assert.Contains("bg-warning", model.BadgeCssClass, StringComparison.Ordinal);
|
||||
Assert.Equal(AlarmProviderMode.Alarmmgr, model.Mode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -719,6 +719,47 @@ public sealed class GatewaySessionTests
|
||||
Assert.Equal([1UL, 2UL], received.Select(mxEvent => mxEvent.WorkerSequence).ToArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-34, the disposal-side twin of the GWC-27 premature-attach poison: an attach that
|
||||
/// passed its readiness gate and then lost the CPU until the session was torn down used to
|
||||
/// find <c>_eventDistributor</c> null, read that as "not created yet" and build a
|
||||
/// REPLACEMENT distributor. Its pump could never start (the started flag is a one-way latch)
|
||||
/// and its channel was never completed, so the subscriber registered on it and blocked in
|
||||
/// <c>WaitToReadAsync</c> until its own deadline — no error, no log, no metric. Disposal now
|
||||
/// latches the null-out, so every attach entry point fails immediately and loudly instead.
|
||||
/// Attaching twice pins the rollback: the second call must still report
|
||||
/// <c>SessionNotReady</c>, not <c>EventSubscriberAlreadyActive</c>, which it only can if the
|
||||
/// first failure released the subscriber slot it took.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AttachAfterDisposeThrowsSessionNotReadyInsteadOfRegisteringOnANeverPumpedDistributor()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
|
||||
// Disposal leaves the session state Ready (only CloseAsync transitions it), so the attach
|
||||
// paths' readiness gates pass and the calls below genuinely reach EnsureDistributorCreated —
|
||||
// the exact position the racing attach in the finding occupies.
|
||||
await session.DisposeAsync();
|
||||
|
||||
SessionManagerException attachException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachEventSubscriber(maxSubscribers: 1));
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, attachException.ErrorCode);
|
||||
|
||||
SessionManagerException secondAttachException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachEventSubscriber(maxSubscribers: 1));
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, secondAttachException.ErrorCode);
|
||||
|
||||
SessionManagerException replayException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachEventSubscriberWithReplay(maxSubscribers: 1, afterSequence: 0));
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, replayException.ErrorCode);
|
||||
|
||||
SessionManagerException internalException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachInternalEventSubscriber());
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, internalException.ErrorCode);
|
||||
}
|
||||
|
||||
private static GatewaySession CreateSession()
|
||||
{
|
||||
return new GatewaySession(
|
||||
|
||||
@@ -14,6 +14,10 @@ public sealed class WorkerClientTests
|
||||
private const string SessionId = "session-worker-client";
|
||||
private const string Nonce = "nonce-worker-client";
|
||||
private const int WorkerProcessId = 4321;
|
||||
|
||||
/// <summary>Upper bound on the events <see cref="EmitEventsUntilStoppedAsync"/> writes.</summary>
|
||||
private const ulong EmittedEventCeiling = 2000;
|
||||
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Verifies that StartAsync enters ready state after receiving worker hello and ready messages.</summary>
|
||||
@@ -705,6 +709,96 @@ public sealed class WorkerClientTests
|
||||
Assert.True(process.Disposed);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-32: disposing a still-<c>Ready</c> client while the worker is emitting events must be
|
||||
/// an ordinary teardown, not a fault. <c>DisposeAsync</c> completes the same three channels
|
||||
/// <c>MarkClosed</c> and <c>SetFaulted</c> do, so it must enter a terminal state first;
|
||||
/// otherwise the read loop — already inside <c>DispatchEnvelope</c> when the staging channel
|
||||
/// is completed, or reading a stream that is torn down underneath it — misdiagnoses the
|
||||
/// completed channel as the GWC-24 sustained-backpressure overflow and raises a spurious
|
||||
/// <c>ProtocolViolation</c> fault, a <c>worker-event-staging</c> overflow metric and a
|
||||
/// redundant kill during an intentional dispose. The event channel is sized far above the
|
||||
/// burst so a genuine overflow is impossible, and <c>DisposeAsync</c> awaits its background
|
||||
/// tasks, so the metrics read afterwards is not racing a late fault.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_WhileWorkerIsEmittingEvents_ClosesWithoutFaultingOrOverflowing()
|
||||
{
|
||||
// The burst is capped at EmittedEventCeiling and the staging channel holds 2 x capacity, so
|
||||
// even if every emitted event were staged and none consumed the GWC-24 bound is unreachable:
|
||||
// an overflow metric in this test can only be the dispose-race misdiagnosis.
|
||||
const int capacity = 2048;
|
||||
using GatewayMetrics metrics = new();
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
EventChannelCapacity = capacity,
|
||||
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||||
},
|
||||
metrics: metrics);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
using CancellationTokenSource emitCts = new(TestTimeout);
|
||||
Task emitTask = EmitEventsUntilStoppedAsync(pipePair, emitCts.Token);
|
||||
|
||||
// Proves the read loop is mid-stream — not parked before its first event — when dispose runs.
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().WorkerEventQueueDepth > 0, TestTimeout);
|
||||
|
||||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||||
await emitCts.CancelAsync();
|
||||
await emitTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Closed, client.State);
|
||||
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0L, snapshot.Faults);
|
||||
Assert.Equal(0L, snapshot.QueueOverflows);
|
||||
Assert.Equal(0L, snapshot.WorkerKills);
|
||||
|
||||
// Disposal is now a terminal transition, so the worker that reached Ready is recorded as
|
||||
// stopped exactly once — the running-worker gauge no longer leaks on this path.
|
||||
Assert.Equal(0, snapshot.WorkersRunning);
|
||||
Assert.Equal(1L, snapshot.WorkerExits);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes worker events from the fake worker side until cancelled or until
|
||||
/// <see cref="EmittedEventCeiling"/> events have been written, so the gateway read loop has a
|
||||
/// continuous stream to dispatch while a test tears the client down. The ceiling keeps the
|
||||
/// burst structurally below the staging bound the caller configures. Every failure is
|
||||
/// swallowed: disposal closes the gateway end of the pipe underneath this writer, which is
|
||||
/// the situation under test rather than a test failure.
|
||||
/// </summary>
|
||||
/// <param name="pipePair">The connected pipe pair whose worker side emits the events.</param>
|
||||
/// <param name="cancellationToken">Stops the emission.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private static async Task EmitEventsUntilStoppedAsync(
|
||||
PipePair pipePair,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Yield();
|
||||
try
|
||||
{
|
||||
for (ulong sequence = 1;
|
||||
sequence <= EmittedEventCeiling && !cancellationToken.IsCancellationRequested;
|
||||
sequence++)
|
||||
{
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange),
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Broken pipe, disposed stream or cancellation — all expected once dispose starts.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a heartbeat envelope updates the last-heartbeat timestamp and worker
|
||||
/// process id. Uses a <see cref="ManualTimeProvider"/> so the timestamp advance is
|
||||
|
||||
Reference in New Issue
Block a user