fix(alarms): truncation-safe transitions; perf: single-pass alarm parse; configurable poll cadence

This commit is contained in:
Joseph Doherty
2026-08-15 17:04:06 -04:00
parent 94fdc18c3c
commit f3e1de5f37
10 changed files with 910 additions and 33 deletions
@@ -19,7 +19,26 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
internal const string WriteCompletionWaitEnvironmentVariableName =
"MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS";
private static readonly TimeSpan AlarmPollInterval = TimeSpan.FromMilliseconds(500);
/// <summary>
/// Environment variable the gateway's WorkerProcessLauncher sets from
/// MxGateway:Alarms:PollIntervalMilliseconds. A missing or invalid
/// value falls back to <see cref="DefaultAlarmPollInterval"/>.
/// </summary>
internal const string AlarmPollIntervalEnvironmentVariableName =
"MXGATEWAY_ALARM_POLL_INTERVAL_MS";
/// <summary>
/// Floor on the resolved alarm poll cadence. Mirrors the gateway-side
/// MxGateway:Alarms:PollIntervalMilliseconds minimum so a value that
/// slipped past validation (or an environment set by hand) still can
/// not starve the STA's command path.
/// </summary>
internal static readonly TimeSpan MinimumAlarmPollInterval = TimeSpan.FromMilliseconds(100);
/// <summary>Default alarm poll cadence when the environment says nothing usable.</summary>
internal static readonly TimeSpan DefaultAlarmPollInterval = TimeSpan.FromMilliseconds(500);
private readonly TimeSpan alarmPollInterval = ResolveAlarmPollInterval();
private readonly IMxAccessComObjectFactory factory;
private readonly IMxAccessEventSink eventSink;
@@ -191,6 +210,31 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
: MxAccessCommandExecutor.DefaultWriteCompletionTimeout;
}
/// <summary>
/// Resolves the alarm poll cadence from the launcher-provided
/// environment variable. A missing, unparseable, or out-of-range value
/// falls back to <see cref="DefaultAlarmPollInterval"/> rather than
/// faulting the worker: an alarm poll that runs at the default cadence
/// is always safe, and a session that refuses to start over a
/// mistyped environment variable is not.
/// </summary>
/// <returns>The cadence the alarm poll loop waits between polls.</returns>
internal static TimeSpan ResolveAlarmPollInterval()
{
string? value = Environment.GetEnvironmentVariable(AlarmPollIntervalEnvironmentVariableName);
if (!int.TryParse(
value,
System.Globalization.NumberStyles.Integer,
System.Globalization.CultureInfo.InvariantCulture,
out int milliseconds))
{
return DefaultAlarmPollInterval;
}
TimeSpan resolved = TimeSpan.FromMilliseconds(milliseconds);
return resolved < MinimumAlarmPollInterval ? DefaultAlarmPollInterval : resolved;
}
/// <summary>
/// Starts the MXAccess COM session asynchronously.
/// </summary>
@@ -274,7 +318,7 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
{
try
{
await Task.Delay(AlarmPollInterval, cancellationToken).ConfigureAwait(false);
await Task.Delay(alarmPollInterval, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Xml;
@@ -49,11 +50,39 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
private const string DefaultVersion = "1.0";
private const int DefaultMaxAlarmsPerFetch = 1024;
/// <summary>
/// Floor on the resolved per-fetch cap. Mirrors the gateway-side
/// <c>MxGateway:Alarms:MaxAlarmsPerFetch</c> minimum so a value that
/// slipped past startup validation still leaves a usable snapshot
/// window.
/// </summary>
internal const int MinimumMaxAlarmsPerFetch = 64;
/// <summary>
/// Environment variable the gateway's <c>WorkerProcessLauncher</c>
/// sets from <c>MxGateway:Alarms:MaxAlarmsPerFetch</c>. A missing,
/// unparseable, or below-floor value falls back to
/// <see cref="DefaultMaxAlarmsPerFetch"/> — a bad environment value
/// must never keep the alarm consumer from starting.
/// </summary>
internal const string MaxAlarmsPerFetchEnvironmentVariableName =
"MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH";
/// <summary>
/// Minimum gap between truncated-fetch warnings. A galaxy parked above
/// the cap truncates on *every* poll, so an unthrottled warning would
/// be two log lines a second forever.
/// </summary>
private const long TruncationWarningIntervalMilliseconds = 60_000;
private readonly object syncRoot = new object();
private readonly Dictionary<Guid, MxAlarmSnapshotRecord> latestSnapshot =
new Dictionary<Guid, MxAlarmSnapshotRecord>();
private readonly int maxAlarmsPerFetch;
private readonly Stopwatch truncationWarningClock = Stopwatch.StartNew();
private long lastTruncationWarningMilliseconds = -TruncationWarningIntervalMilliseconds;
private long truncatedFetchCount;
private wwAlarmConsumerClass? client;
private wwAlarmConsumerClass? ackClient;
private bool subscribed;
@@ -67,7 +96,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
/// every COM call stays on the STA that owns the apartment.
/// </summary>
public WnWrapAlarmConsumer()
: this(new wwAlarmConsumerClass(), DefaultMaxAlarmsPerFetch)
: this(new wwAlarmConsumerClass(), ResolveMaxAlarmsPerFetch())
{
}
@@ -86,6 +115,27 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
: DefaultMaxAlarmsPerFetch;
}
/// <summary>
/// Resolves the per-fetch cap from the launcher-provided environment
/// variable. A missing, unparseable, or below-floor value falls back
/// to <see cref="DefaultMaxAlarmsPerFetch"/> rather than throwing:
/// polling at the default cap is always safe, and a session that
/// refuses to subscribe over a mistyped environment variable is not.
/// </summary>
/// <returns>The cap passed to <c>GetXmlCurrentAlarms2</c>.</returns>
internal static int ResolveMaxAlarmsPerFetch()
{
string? value = Environment.GetEnvironmentVariable(MaxAlarmsPerFetchEnvironmentVariableName);
return int.TryParse(
value,
NumberStyles.Integer,
CultureInfo.InvariantCulture,
out int cap)
&& cap >= MinimumMaxAlarmsPerFetch
? cap
: DefaultMaxAlarmsPerFetch;
}
/// <summary>
/// Fires once per detected alarm-state transition (raise, acknowledge,
/// clear, or new-alarm-already-acked-on-arrival), dispatched from
@@ -282,6 +332,15 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
}
/// <inheritdoc />
/// <remarks>
/// This is the set the gateway's reconcile pass diffs its cache
/// against, so an alarm missing here is what the gateway reads as a
/// Clear. After a truncated fetch the snapshot deliberately still
/// carries alarms the capped reply could not mention — see
/// <see cref="PollOnce"/>. Erring toward "still active" keeps the feed
/// at-least-once: a stale entry is repaired by the next sub-cap poll,
/// whereas a dropped one is broadcast as a Clear that never happened.
/// </remarks>
public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms()
{
if (disposed) throw new ObjectDisposedException(nameof(WnWrapAlarmConsumer));
@@ -300,6 +359,16 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
}
}
/// <summary>
/// Sink for the rate-limited truncated-fetch warning. Defaults to
/// <see cref="Console.Error"/>, the stream
/// <c>WorkerConsoleLogger</c> already writes to, so the line lands in
/// the worker's captured stderr alongside the rest of its log. Tests
/// substitute a collector. The message carries identifiers and counts
/// only — never alarm values, tag names, or comments.
/// </summary>
internal Action<string>? TruncationWarningSink { get; set; }
/// <inheritdoc />
public void PollOnce()
{
@@ -316,17 +385,28 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
string xml = xmlObj?.ToString() ?? string.Empty;
if (xml.Length == 0) return;
Dictionary<Guid, MxAlarmSnapshotRecord> next = ParseSnapshotXml(xml);
Dictionary<Guid, MxAlarmSnapshotRecord> next = ParseSnapshotXml(xml, out int fetchedRecordCount);
// TRUNCATION CLIFF. GetXmlCurrentAlarms2 caps its reply at maxAlmCnt
// and gives no "there is more" flag, so a reply holding exactly the
// cap is indistinguishable from a galaxy that happens to have exactly
// that many active alarms. Treat the ambiguous case as truncated: the
// false-positive cost is a snapshot that stays stale for one poll, the
// false-negative cost is every alarm past the cap reading as cleared.
bool truncated = IsTruncatedFetch(fetchedRecordCount, maxAlarmsPerFetch);
IReadOnlyList<MxAlarmTransitionEvent> transitions;
int retainedCount;
lock (syncRoot)
{
transitions = ComputeTransitions(latestSnapshot, next);
latestSnapshot.Clear();
foreach (KeyValuePair<Guid, MxAlarmSnapshotRecord> kv in next)
{
latestSnapshot[kv.Key] = kv.Value;
}
ApplySnapshotUpdate(latestSnapshot, next, truncated);
retainedCount = latestSnapshot.Count;
}
if (truncated)
{
WarnTruncatedFetch(fetchedRecordCount, next.Count, retainedCount);
}
if (transitions.Count == 0) return;
@@ -338,6 +418,111 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
}
}
/// <summary>
/// Decides whether a fetch that came back holding
/// <paramref name="fetchedRecordCount"/> records hit the cap.
/// <c>GetXmlCurrentAlarms2</c> caps its reply at <c>maxAlmCnt</c> and
/// offers no "more available" flag, so a reply at exactly the cap is
/// indistinguishable from a galaxy that happens to hold exactly that
/// many active alarms; both are treated as truncated. Exposed as
/// <c>internal static</c> so the rule is unit-testable without the
/// wnwrapConsumer COM object.
/// </summary>
/// <param name="fetchedRecordCount">ALARM records the reply carried.</param>
/// <param name="maxAlarmsPerFetch">The cap that was passed to the fetch.</param>
/// <returns><see langword="true"/> when the reply must be treated as truncated.</returns>
internal static bool IsTruncatedFetch(int fetchedRecordCount, int maxAlarmsPerFetch)
{
return fetchedRecordCount >= maxAlarmsPerFetch;
}
/// <summary>
/// Folds a freshly fetched snapshot into the retained one.
/// </summary>
/// <remarks>
/// <para>
/// A complete fetch is authoritative about <em>absence</em>:
/// replacing the snapshot wholesale is exactly what lets a cleared
/// alarm drop out of <see cref="SnapshotActiveAlarms"/>, which is
/// the evidence the gateway's reconcile pass turns into a Clear.
/// </para>
/// <para>
/// A truncated fetch is authoritative about <em>presence</em>
/// only. Merging rather than replacing means alarms the capped
/// reply did carry update normally, while alarms it had no room to
/// mention are retained untouched — so the gateway infers no Clear
/// from their absence. The cost is bounded staleness: a genuinely
/// cleared alarm can linger in the snapshot until the first
/// sub-cap fetch evicts it. That direction is the safe one for an
/// at-least-once alarm feed — a late Clear is repaired by the next
/// complete poll, whereas a Clear that never happened is broadcast
/// to every subscriber and cannot be taken back.
/// </para>
/// </remarks>
/// <param name="snapshot">The retained snapshot, updated in place.</param>
/// <param name="next">The snapshot just parsed from the fetch.</param>
/// <param name="truncated">Whether the fetch hit the per-fetch cap.</param>
internal static void ApplySnapshotUpdate(
Dictionary<Guid, MxAlarmSnapshotRecord> snapshot,
Dictionary<Guid, MxAlarmSnapshotRecord> next,
bool truncated)
{
if (snapshot is null) throw new ArgumentNullException(nameof(snapshot));
if (next is null) throw new ArgumentNullException(nameof(next));
if (!truncated)
{
snapshot.Clear();
}
foreach (KeyValuePair<Guid, MxAlarmSnapshotRecord> kv in next)
{
snapshot[kv.Key] = kv.Value;
}
}
/// <summary>
/// Emits the truncated-fetch warning at most once per
/// <see cref="TruncationWarningIntervalMilliseconds"/>, counting every
/// truncated poll in between so the operator can tell a one-off burst
/// from a galaxy permanently parked above the cap.
/// </summary>
/// <param name="fetchedRecordCount">ALARM records the capped reply carried.</param>
/// <param name="parsedRecordCount">Records that survived GUID parsing.</param>
/// <param name="retainedCount">Snapshot size after the merge.</param>
private void WarnTruncatedFetch(int fetchedRecordCount, int parsedRecordCount, int retainedCount)
{
long total;
long elapsed;
lock (syncRoot)
{
total = ++truncatedFetchCount;
elapsed = truncationWarningClock.ElapsedMilliseconds;
if (elapsed - lastTruncationWarningMilliseconds < TruncationWarningIntervalMilliseconds)
{
return;
}
lastTruncationWarningMilliseconds = elapsed;
}
// Identifiers and counts only — no tag names, values, limits, or
// comments, per the gateway's "don't log tag values by default" rule.
string message = string.Format(
CultureInfo.InvariantCulture,
"level=Warning event=AlarmSnapshotTruncated maxAlarmsPerFetch={0} fetchedRecords={1} "
+ "parsedRecords={2} retainedSnapshotSize={3} truncatedFetchesSinceStart={4} "
+ "note=absence-implies-clear suppressed for this poll; raise MxGateway:Alarms:MaxAlarmsPerFetch",
maxAlarmsPerFetch,
fetchedRecordCount,
parsedRecordCount,
retainedCount,
total);
Action<string> sink = TruncationWarningSink ?? Console.Error.WriteLine;
sink(message);
}
/// <summary>
/// Pure snapshot-to-transitions diff. Compares the previous polled
/// snapshot to the next snapshot and produces one
@@ -355,6 +540,21 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
/// <item><description>A GUID present in both with a different <see cref="MxAlarmSnapshotRecord.State"/> produces a transition carrying the prior state.</description></item>
/// <item><description>A GUID present in <paramref name="previous"/> but absent from <paramref name="next"/> produces no transition. AVEVA drops cleared alarms from the active set; the snapshot simply stops mentioning them.</description></item>
/// </list>
/// <para>
/// The absence rule is deliberately inert here: this diff never
/// turns a disappearance into a Clear. The clear is inferred one
/// level up, by the gateway's reconcile pass diffing its cache
/// against <see cref="SnapshotActiveAlarms"/>, which reads the
/// snapshot <see cref="PollOnce"/> maintains from these same
/// dictionaries. That is why the truncation guard lives in
/// <see cref="PollOnce"/>'s snapshot update rather than in this
/// method: suppressing "absence implies Clear" means declining to
/// *evict* alarms a capped fetch could not mention, not declining
/// to emit a transition this method was never going to emit.
/// Passing a truncated <paramref name="next"/> straight through
/// stays correct for the alarms it does carry — first sightings
/// and state changes are computed from presence alone.
/// </para>
/// </remarks>
/// <param name="previous">The snapshot from the previous poll (or empty on first call).</param>
/// <param name="next">The snapshot just parsed from <c>GetXmlCurrentAlarms2</c>.</param>
@@ -393,9 +593,29 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
/// <param name="xml">The XML snapshot payload.</param>
/// <returns>The parsed alarm snapshot records, keyed by alarm GUID.</returns>
public static Dictionary<Guid, MxAlarmSnapshotRecord> ParseSnapshotXml(string xml)
{
return ParseSnapshotXml(xml, out _);
}
/// <summary>
/// <see cref="ParseSnapshotXml(string)"/> plus the raw
/// <c>ALARM</c>-element count, which <see cref="PollOnce"/> compares
/// against the per-fetch cap to detect a truncated reply. The raw
/// count — not the dictionary size — is the right signal: records
/// dropped for a malformed GUID still consumed a slot in the capped
/// reply, so counting only the survivors would under-report and let a
/// truncated fetch pass as complete.
/// </summary>
/// <param name="xml">The XML snapshot payload.</param>
/// <param name="alarmRecordCount">Number of <c>ALARM</c> elements in the payload.</param>
/// <returns>The parsed alarm snapshot records, keyed by alarm GUID.</returns>
internal static Dictionary<Guid, MxAlarmSnapshotRecord> ParseSnapshotXml(
string xml,
out int alarmRecordCount)
{
Dictionary<Guid, MxAlarmSnapshotRecord> records =
new Dictionary<Guid, MxAlarmSnapshotRecord>();
alarmRecordCount = 0;
if (string.IsNullOrWhiteSpace(xml)) return records;
@@ -404,45 +624,96 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
XmlNodeList? alarmNodes = doc.SelectNodes("/ALARM_RECORDS/ALARM");
if (alarmNodes is null) return records;
alarmRecordCount = alarmNodes.Count;
foreach (XmlNode alarmNode in alarmNodes)
{
string guidHex = TextOf(alarmNode, "GUID");
// One pass over the child elements instead of 17 SelectSingleNode
// XPath evaluations per record. At the 1024-record cap that is the
// difference between ~17k XPath expression compilations plus child
// walks and one walk per record, on the STA that also serves reads
// and writes.
//
// Parity with the XPath it replaces: SelectSingleNode(name) picks
// the FIRST child ELEMENT of that name (ordinal, case-sensitive)
// and yields its InnerText; an absent child yields string.Empty,
// and so does a present-but-empty one. Hence: element nodes only,
// first-match-wins (null means "not seen yet"), and a single
// coalesce to string.Empty at the end — which keeps the absent and
// empty cases indistinguishable exactly as before.
string? guidHex = null;
string? xmlDate = null;
string? xmlTime = null;
string? gmtOffset = null;
string? dstAdjust = null;
string? providerNode = null;
string? providerName = null;
string? group = null;
string? tagName = null;
string? type = null;
string? value = null;
string? limit = null;
string? priority = null;
string? state = null;
string? operatorNode = null;
string? operatorName = null;
string? alarmComment = null;
foreach (XmlNode child in alarmNode.ChildNodes)
{
if (child.NodeType != XmlNodeType.Element) continue;
switch (child.Name)
{
case "GUID": guidHex ??= child.InnerText; break;
case "DATE": xmlDate ??= child.InnerText; break;
case "TIME": xmlTime ??= child.InnerText; break;
case "GMTOFFSET": gmtOffset ??= child.InnerText; break;
case "DSTADJUST": dstAdjust ??= child.InnerText; break;
case "PROVIDER_NODE": providerNode ??= child.InnerText; break;
case "PROVIDER_NAME": providerName ??= child.InnerText; break;
case "GROUP": group ??= child.InnerText; break;
case "TAGNAME": tagName ??= child.InnerText; break;
case "TYPE": type ??= child.InnerText; break;
case "VALUE": value ??= child.InnerText; break;
case "LIMIT": limit ??= child.InnerText; break;
case "PRIORITY": priority ??= child.InnerText; break;
case "STATE": state ??= child.InnerText; break;
case "OPERATOR_NODE": operatorNode ??= child.InnerText; break;
case "OPERATOR_NAME": operatorName ??= child.InnerText; break;
case "ALARM_COMMENT": alarmComment ??= child.InnerText; break;
}
}
if (!TryParseHexGuid(guidHex, out Guid guid)) continue;
string xmlDate = TextOf(alarmNode, "DATE");
string xmlTime = TextOf(alarmNode, "TIME");
int gmtOffset = ParseInt(TextOf(alarmNode, "GMTOFFSET"));
int dstAdjust = ParseInt(TextOf(alarmNode, "DSTADJUST"));
DateTime tsUtc = AlarmRecordTransitionMapper.ParseTransitionTimestampUtc(
xmlDate, xmlTime, gmtOffset, dstAdjust);
xmlDate ?? string.Empty,
xmlTime ?? string.Empty,
ParseInt(gmtOffset ?? string.Empty),
ParseInt(dstAdjust ?? string.Empty));
records[guid] = new MxAlarmSnapshotRecord
{
AlarmGuid = guid,
TransitionTimestampUtc = tsUtc,
ProviderNode = TextOf(alarmNode, "PROVIDER_NODE"),
ProviderName = TextOf(alarmNode, "PROVIDER_NAME"),
Group = TextOf(alarmNode, "GROUP"),
TagName = TextOf(alarmNode, "TAGNAME"),
Type = TextOf(alarmNode, "TYPE"),
Value = TextOf(alarmNode, "VALUE"),
Limit = TextOf(alarmNode, "LIMIT"),
Priority = ParseInt(TextOf(alarmNode, "PRIORITY")),
State = AlarmRecordTransitionMapper.ParseStateKind(TextOf(alarmNode, "STATE")),
OperatorNode = TextOf(alarmNode, "OPERATOR_NODE"),
OperatorName = TextOf(alarmNode, "OPERATOR_NAME"),
AlarmComment = TextOf(alarmNode, "ALARM_COMMENT"),
ProviderNode = providerNode ?? string.Empty,
ProviderName = providerName ?? string.Empty,
Group = group ?? string.Empty,
TagName = tagName ?? string.Empty,
Type = type ?? string.Empty,
Value = value ?? string.Empty,
Limit = limit ?? string.Empty,
Priority = ParseInt(priority ?? string.Empty),
State = AlarmRecordTransitionMapper.ParseStateKind(state ?? string.Empty),
OperatorNode = operatorNode ?? string.Empty,
OperatorName = operatorName ?? string.Empty,
AlarmComment = alarmComment ?? string.Empty,
};
}
return records;
}
private static string TextOf(XmlNode parent, string childName)
{
XmlNode? node = parent.SelectSingleNode(childName);
return node?.InnerText ?? string.Empty;
}
private static int ParseInt(string text)
{
return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int n)