feat(alarms): structural degraded-status signal for truncated alarm snapshots
The truncation-cliff fix made alarm transitions truncation-safe but silent:
when GetXmlCurrentAlarms2 returns exactly maxAlmCnt records the worker
suppresses absence-implies-Clear inference and says so only in a rate-limited
stderr warning. No client and no operator could tell a complete active set
from a capped one.
Two additive proto3 booleans carry the verdict out:
- QueryActiveAlarmsReplyPayload.snapshot_truncated = 2 (worker IPC reply)
- ActiveAlarmSnapshot.from_truncated_snapshot = 16 (per record)
The per-record field is not an aesthetic choice. QueryActiveAlarms returns a
bare `stream ActiveAlarmSnapshot` with no envelope, header, or trailer, so a
per-record boolean is the only carrier that stays wire-compatible; an envelope
message would change every existing client's stream element type. The reply
payload states it too because a prefix filter can leave zero records and a
truncated fetch with nothing to report still has to say so. The flag means
"this set may be incomplete", never "this record is unreliable" — it is
independent of the subtag-fallback `degraded` field.
Detection is deliberately UNCHANGED: IsTruncatedFetch remains
`fetchedRecordCount >= maxAlarmsPerFetch`. The live probe (docs/AlarmProbeFindings.md,
ce5d8ae) could not verify whether ALARM_RECORDS/@COUNT reports the total active
count or only the records in the reply, so @COUNT is not parsed for detection;
switching to it stays blocked on probe evidence. The probe's comment
annotations in WnWrapAlarmConsumer.cs are preserved.
Reset semantics: not latched. WnWrapAlarmConsumer.FoldFetch replaces the
verdict on every poll under the same lock as the snapshot merge, so the first
sub-cap fetch clears it; GatewayAlarmMonitor.ClearCache drops it with the cache
generation it describes. A caveat that never turns off is one operators learn
to ignore.
Flow: WnWrapAlarmConsumer.LastSnapshotTruncated -> AlarmDispatcher (stamps every
record) / IAlarmCommandHandler (payload) -> MxAccessCommandExecutor reply ->
GatewayAlarmMonitor._snapshotTruncated -> IGatewayAlarmService.SnapshotTruncated
-> DashboardAlarmQueryResult -> AlarmsPage warning banner (render-side only; the
poll loop and DisposeAsync drain are untouched). The public QueryActiveAlarms
RPC forwards worker snapshots unmodified, so the per-record flag needed no
mapper change — a test pins that.
Parity: this describes OUR fetch mechanics — additive gateway metadata — not
MXAccess provider behavior. No event is synthesized and no MXAccess-observable
semantics change, so it is not a parity deviation.
Tests: worker LastSnapshotTruncated set/reset/consecutive-burst (windev-run);
gateway end-to-end truncated reply -> monitor -> public stream, with the
complete-reply control as the load-bearing assertion; AlarmsPage banner
present/absent. Docs: gateway.md alarm surface, docs/DesignDecisions.md entry.
This commit is contained in:
@@ -342,6 +342,25 @@ public sealed class AlarmCommandHandler : IAlarmCommandHandler
|
||||
return filtered;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Deliberately does not go through <c>GetDispatcherOrThrow</c>: an
|
||||
/// unsubscribed handler has performed no fetch, and "no fetch" is not
|
||||
/// truncated. Throwing here would turn a status read into a command
|
||||
/// failure on a path the reply builder takes after the snapshot has
|
||||
/// already been produced.
|
||||
/// </remarks>
|
||||
public bool LastSnapshotTruncated
|
||||
{
|
||||
get
|
||||
{
|
||||
if (disposed) return false;
|
||||
AlarmDispatcher? d;
|
||||
lock (syncRoot) d = dispatcher;
|
||||
return d is not null && d.LastSnapshotTruncated;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void PollOnce()
|
||||
{
|
||||
|
||||
@@ -158,16 +158,28 @@ public sealed class AlarmDispatcher : IDisposable
|
||||
public IReadOnlyList<ActiveAlarmSnapshot> SnapshotActiveAlarms()
|
||||
{
|
||||
if (disposed) throw new ObjectDisposedException(nameof(AlarmDispatcher));
|
||||
// Read the truncation verdict before the snapshot, so a poll landing
|
||||
// between the two can only widen the warning (a stale "truncated" over a
|
||||
// complete snapshot), never narrow it into a false all-clear.
|
||||
bool truncated = consumer.LastSnapshotTruncated;
|
||||
IReadOnlyList<MxAlarmSnapshotRecord> records = consumer.SnapshotActiveAlarms();
|
||||
if (records.Count == 0) return Array.Empty<ActiveAlarmSnapshot>();
|
||||
List<ActiveAlarmSnapshot> snapshots = new List<ActiveAlarmSnapshot>(records.Count);
|
||||
foreach (MxAlarmSnapshotRecord record in records)
|
||||
{
|
||||
snapshots.Add(MapToSnapshot(record));
|
||||
snapshots.Add(MapToSnapshot(record, truncated));
|
||||
}
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the consumer's most recent fetch hit the per-fetch cap, so
|
||||
/// the set <see cref="SnapshotActiveAlarms"/> returns may omit actives.
|
||||
/// Stamped onto the QueryActiveAlarms reply payload, which is the only
|
||||
/// carrier when the snapshot filters down to zero records.
|
||||
/// </summary>
|
||||
public bool LastSnapshotTruncated => !disposed && consumer.LastSnapshotTruncated;
|
||||
|
||||
private void OnTransition(object? sender, MxAlarmTransitionEvent transition)
|
||||
{
|
||||
if (disposed) return;
|
||||
@@ -196,7 +208,7 @@ public sealed class AlarmDispatcher : IDisposable
|
||||
degraded: record.Degraded);
|
||||
}
|
||||
|
||||
private static ActiveAlarmSnapshot MapToSnapshot(MxAlarmSnapshotRecord record)
|
||||
private static ActiveAlarmSnapshot MapToSnapshot(MxAlarmSnapshotRecord record, bool truncated)
|
||||
{
|
||||
ActiveAlarmSnapshot snapshot = new ActiveAlarmSnapshot
|
||||
{
|
||||
@@ -212,6 +224,12 @@ public sealed class AlarmDispatcher : IDisposable
|
||||
Description = string.Empty,
|
||||
Degraded = record.Degraded,
|
||||
SourceProvider = record.Degraded ? AlarmProviderMode.Subtag : AlarmProviderMode.Alarmmgr,
|
||||
// Set-level status, stamped identically on every record of the
|
||||
// snapshot: QueryActiveAlarms streams bare ActiveAlarmSnapshot
|
||||
// messages with no envelope to hang it off. Independent of
|
||||
// Degraded above — that is about this record's provider, this is
|
||||
// about whether the set it belongs to is complete.
|
||||
FromTruncatedSnapshot = truncated,
|
||||
};
|
||||
if (record.TransitionTimestampUtc != DateTime.MinValue)
|
||||
{
|
||||
|
||||
@@ -266,6 +266,17 @@ public sealed class FailoverAlarmConsumer : IMxAccessAlarmConsumer
|
||||
return ActiveChild.SnapshotActiveAlarms();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Delegated to the active child, matching
|
||||
/// <see cref="SnapshotActiveAlarms"/>: the flag describes the snapshot
|
||||
/// the same child produced, so reading it off the standby would pair a
|
||||
/// verdict with a snapshot it does not belong to. A failover to the
|
||||
/// subtag standby therefore reports not-truncated — correctly, since
|
||||
/// that child performs no capped fetch.
|
||||
/// </remarks>
|
||||
public bool LastSnapshotTruncated => !disposed && ActiveChild.LastSnapshotTruncated;
|
||||
|
||||
private IMxAccessAlarmConsumer ActiveChild => active == Active.Primary ? primary : standby;
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -74,6 +74,16 @@ public interface IAlarmCommandHandler : IDisposable
|
||||
/// <returns>The currently active alarms matching the filter.</returns>
|
||||
IReadOnlyList<ActiveAlarmSnapshot> QueryActive(string? alarmFilterPrefix);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the consumer's most recent fetch hit the per-fetch cap, so
|
||||
/// the set <see cref="QueryActive"/> draws from may omit active alarms.
|
||||
/// Stamped on the QueryActiveAlarms reply payload — the only carrier
|
||||
/// once a prefix filter (or an empty galaxy) leaves zero records to
|
||||
/// carry the per-record flag. <see langword="false"/> when there is no
|
||||
/// active subscription: no fetch has happened, so nothing is capped.
|
||||
/// </summary>
|
||||
bool LastSnapshotTruncated { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Drives a single poll of the underlying alarm consumer on the
|
||||
/// caller's thread. This is a no-op when there is no active
|
||||
|
||||
@@ -36,6 +36,20 @@ public interface IMxAccessAlarmConsumer : IDisposable
|
||||
/// </summary>
|
||||
event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the most recent fetch that reached the retained snapshot came
|
||||
/// back holding the per-fetch cap. While this is <see langword="true"/>
|
||||
/// the snapshot returned by <see cref="SnapshotActiveAlarms"/> is
|
||||
/// authoritative about presence only: the provider may hold actives it
|
||||
/// had no room to report, and the consumer has suspended the
|
||||
/// absence-implies-Clear inference. Not latched — the first sub-cap fetch
|
||||
/// after a run of capped ones clears it, because that fetch is complete
|
||||
/// and the snapshot it produced is again authoritative about absence.
|
||||
/// Consumers with no per-fetch cap (the subtag fallback, which is
|
||||
/// event-driven) always report <see langword="false"/>.
|
||||
/// </summary>
|
||||
bool LastSnapshotTruncated { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the AVEVA alarm-client connection, registers as a
|
||||
/// consumer, and subscribes to the supplied alarm-provider expression.
|
||||
|
||||
@@ -977,6 +977,10 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
|
||||
command.Command.QueryActiveAlarmsCommand.AlarmFilterPrefix);
|
||||
QueryActiveAlarmsReplyPayload payload = new QueryActiveAlarmsReplyPayload();
|
||||
payload.Snapshots.AddRange(snapshots);
|
||||
// Set-level degraded status: the snapshot may omit actives because
|
||||
// the provider fetch hit its cap. The records carry the same flag,
|
||||
// but a prefix filter can leave none, so the payload states it too.
|
||||
payload.SnapshotTruncated = alarmCommandHandler.LastSnapshotTruncated;
|
||||
MxCommandReply reply = CreateOkReply(command);
|
||||
reply.QueryActiveAlarms = payload;
|
||||
return reply;
|
||||
|
||||
@@ -45,6 +45,16 @@ public sealed class SubtagAlarmConsumer : IMxAccessAlarmConsumer
|
||||
/// <summary>Fires once per synthesized alarm-state transition.</summary>
|
||||
public event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Always <see langword="false"/>. Subtag mode is advise-driven over a
|
||||
/// fixed watch list — there is no bulk fetch and therefore no per-fetch
|
||||
/// cap to hit. Subtag snapshots are lower-fidelity in other ways, which
|
||||
/// <c>MxAlarmSnapshotRecord.Degraded</c> already reports; truncation is
|
||||
/// not one of them.
|
||||
/// </remarks>
|
||||
public bool LastSnapshotTruncated => false;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the consumer over a subtag source and a watch list of
|
||||
/// alarm targets.
|
||||
|
||||
@@ -93,6 +93,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
|
||||
|
||||
private long lastTruncationWarningMilliseconds = -TruncationWarningIntervalMilliseconds;
|
||||
private long truncatedFetchCount;
|
||||
private bool lastSnapshotTruncated;
|
||||
private wwAlarmConsumerClass? client;
|
||||
private wwAlarmConsumerClass? ackClient;
|
||||
private bool subscribed;
|
||||
@@ -125,6 +126,23 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
|
||||
: DefaultMaxAlarmsPerFetch;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// COM-free construction, for exercising the retained-snapshot state
|
||||
/// machine (<see cref="FoldFetch"/> / <see cref="LastSnapshotTruncated"/>
|
||||
/// / <see cref="SnapshotActiveAlarms"/>) on a machine without AVEVA
|
||||
/// installed. <see cref="Subscribe"/> throws and <see cref="PollOnce"/>
|
||||
/// no-ops on an instance built this way — both need the wnwrap coclass,
|
||||
/// which cannot be instantiated on the macOS/Linux test matrix. Internal
|
||||
/// rather than public so it cannot be reached from production wiring.
|
||||
/// </summary>
|
||||
/// <param name="maxAlarmsPerFetch">Maximum alarms per fetch call.</param>
|
||||
internal WnWrapAlarmConsumer(int maxAlarmsPerFetch)
|
||||
{
|
||||
this.maxAlarmsPerFetch = maxAlarmsPerFetch > 0
|
||||
? maxAlarmsPerFetch
|
||||
: DefaultMaxAlarmsPerFetch;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the per-fetch cap from the launcher-provided environment
|
||||
/// variable. A missing, unparseable, or out-of-range value falls back
|
||||
@@ -373,6 +391,18 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Read without the disposed guard <see cref="SnapshotActiveAlarms"/>
|
||||
/// carries: this is degraded-status metadata a reply builder stamps
|
||||
/// alongside a snapshot, and throwing from it would fail a query whose
|
||||
/// snapshot half succeeded.
|
||||
/// </remarks>
|
||||
public bool LastSnapshotTruncated
|
||||
{
|
||||
get { lock (syncRoot) { return lastSnapshotTruncated; } }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sink for the rate-limited truncated-fetch warning. Defaults to
|
||||
/// <see cref="Console.Error"/>, the stream
|
||||
@@ -413,14 +443,8 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
|
||||
// docs/AlarmProbeFindings.md.)
|
||||
bool truncated = IsTruncatedFetch(fetchedRecordCount, maxAlarmsPerFetch);
|
||||
|
||||
IReadOnlyList<MxAlarmTransitionEvent> transitions;
|
||||
int retainedCount;
|
||||
lock (syncRoot)
|
||||
{
|
||||
transitions = ComputeTransitions(latestSnapshot, next);
|
||||
ApplySnapshotUpdate(latestSnapshot, next, truncated);
|
||||
retainedCount = latestSnapshot.Count;
|
||||
}
|
||||
IReadOnlyList<MxAlarmTransitionEvent> transitions =
|
||||
FoldFetch(next, truncated, out int retainedCount);
|
||||
|
||||
if (truncated)
|
||||
{
|
||||
@@ -436,6 +460,37 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Folds one fetch into the retained state under a single lock: the
|
||||
/// transition diff, the snapshot merge/replace, and the truncation
|
||||
/// verdict move together. Splitting them would let a concurrent
|
||||
/// <see cref="SnapshotActiveAlarms"/> / <see cref="LastSnapshotTruncated"/>
|
||||
/// pair read a capped snapshot alongside the previous poll's "complete"
|
||||
/// verdict — precisely the false all-clear the signal exists to prevent.
|
||||
/// The verdict is replaced, never latched: a sub-cap fetch is complete
|
||||
/// and restores absence authority, so leaving the flag set would strand
|
||||
/// the operator banner on after a single burst.
|
||||
/// </summary>
|
||||
/// <param name="next">The snapshot just parsed from the fetch.</param>
|
||||
/// <param name="truncated">Whether the fetch hit the per-fetch cap.</param>
|
||||
/// <param name="retainedCount">Size of the retained snapshot after the fold.</param>
|
||||
/// <returns>The transitions the fetch implies.</returns>
|
||||
internal IReadOnlyList<MxAlarmTransitionEvent> FoldFetch(
|
||||
Dictionary<Guid, MxAlarmSnapshotRecord> next,
|
||||
bool truncated,
|
||||
out int retainedCount)
|
||||
{
|
||||
lock (syncRoot)
|
||||
{
|
||||
IReadOnlyList<MxAlarmTransitionEvent> transitions =
|
||||
ComputeTransitions(latestSnapshot, next);
|
||||
ApplySnapshotUpdate(latestSnapshot, next, truncated);
|
||||
lastSnapshotTruncated = truncated;
|
||||
retainedCount = latestSnapshot.Count;
|
||||
return transitions;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decides whether a fetch that came back holding
|
||||
/// <paramref name="fetchedRecordCount"/> records hit the cap.
|
||||
|
||||
Reference in New Issue
Block a user