dohertj2/mxaccessgw
1
0
Fork 0
Code Issues 14 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(alarms): fetch/poll ceilings; truncation-semantics docs; log-format conformance

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-15 17:19:41 -04:00
parent 7c9add3d73
commit b5ea6bb461
11 changed files with 254 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs
+15 -11
View File
@@ -49,22 +49,26 @@ public sealed class AlarmsOptions
/// <summary>
/// Cadence at which the worker's STA polls the AVEVA alarm consumer
/// (<c>GetXmlCurrentAlarms2</c>) for the current active-alarm snapshot.
/// Default 500 ms; must be at least 100 ms. Every poll is a COM call
/// plus an XML parse on the STA that also serves reads and writes, so
/// driving it below 100 ms starves the command path. Conveyed to the
/// worker through the <c>MXGATEWAY_ALARM_POLL_INTERVAL_MS</c>
/// environment variable.
/// Default 500 ms; must be between 100 ms and 3,600,000 ms (one hour).
/// Every poll is a COM call plus an XML parse on the STA that also
/// serves reads and writes, so driving it below 100 ms starves the
/// command path; above an hour the cadence stops being a cadence and
/// silently disables alarm polling. Conveyed to the worker through the
/// <c>MXGATEWAY_ALARM_POLL_INTERVAL_MS</c> environment variable.
/// </summary>
public int PollIntervalMilliseconds { get; init; } = 500;
/// <summary>
/// Cap the worker passes to <c>GetXmlCurrentAlarms2</c>'s
/// <c>maxAlmCnt</c> argument. Default 1024; must be at least 64. A
/// fetch that comes back holding exactly this many records is treated
/// as truncated: the worker keeps the alarms the capped fetch could
/// not mention in its snapshot rather than letting their absence read
/// as a clear. Raise it on galaxies whose steady-state active-alarm
/// count approaches the cap. Conveyed to the worker through the
/// <c>maxAlmCnt</c> argument. Default 1024; must be between 64 and
/// 65,536 — the worker is a 32-bit process that materializes each
/// fetch as one BSTR plus a full XmlDocument, so an unbounded cap
/// faults the STA rather than merely slowing it. A fetch that comes
/// back holding exactly this many records is treated as truncated: the
/// worker keeps the alarms the capped fetch could not mention in its
/// snapshot rather than letting their absence read as a clear. Raise it
/// on galaxies whose steady-state active-alarm count approaches the
/// cap. Conveyed to the worker through the
/// <c>MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH</c> environment variable.
/// </summary>
public int MaxAlarmsPerFetch { get; init; } = 1024;
src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs
+16 -4
View File
@@ -427,23 +427,35 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
private static readonly string[] ValidAlarmFallbackModes = ["Auto", "ForceAlarmManager", "ForceSubtag"];
private const int MinimumAlarmPollIntervalMilliseconds = 100;
// One hour. Above this the cadence stops being a cadence: int.MaxValue
// milliseconds is ~24 days, which silently disables alarm polling instead
// of reporting the misconfiguration.
private const int MaximumAlarmPollIntervalMilliseconds = 3_600_000;
private const int MinimumMaxAlarmsPerFetch = 64;
// The worker is a 32-bit process and materializes each fetch as one BSTR
// plus a full XmlDocument over it, so an unbounded cap is an out-of-memory
// fault on the STA rather than a slow poll.
private const int MaximumMaxAlarmsPerFetch = 65_536;
private static void ValidateAlarms(AlarmsOptions options, ValidationBuilder builder)
{
// Validated regardless of Enabled: both values are stamped onto every
// worker launch environment, so a bad value is a misconfiguration even
// before the central monitor is switched on.
if (options.PollIntervalMilliseconds < MinimumAlarmPollIntervalMilliseconds)
if (options.PollIntervalMilliseconds is < MinimumAlarmPollIntervalMilliseconds
or > MaximumAlarmPollIntervalMilliseconds)
{
builder.Add(
$"MxGateway:Alarms:PollIntervalMilliseconds must be greater than or equal to {MinimumAlarmPollIntervalMilliseconds}.");
$"MxGateway:Alarms:PollIntervalMilliseconds must be between {MinimumAlarmPollIntervalMilliseconds} and {MaximumAlarmPollIntervalMilliseconds}.");
}
if (options.MaxAlarmsPerFetch < MinimumMaxAlarmsPerFetch)
if (options.MaxAlarmsPerFetch is < MinimumMaxAlarmsPerFetch or > MaximumMaxAlarmsPerFetch)
{
builder.Add(
$"MxGateway:Alarms:MaxAlarmsPerFetch must be greater than or equal to {MinimumMaxAlarmsPerFetch}.");
$"MxGateway:Alarms:MaxAlarmsPerFetch must be between {MinimumMaxAlarmsPerFetch} and {MaximumMaxAlarmsPerFetch}.");
}
if (!options.Enabled)
src/ZB.MOM.WW.MxGateway.Server/appsettings.json
+3 -1
View File
@@ -86,7 +86,9 @@
"Enabled": true,
"SubscriptionExpression": "\\\\DESKTOP-6JL3KKO\\Galaxy!DEV",
"DefaultArea": "",
"ReconcileIntervalSeconds": 30
"ReconcileIntervalSeconds": 30,
"PollIntervalMilliseconds": 500,
"MaxAlarmsPerFetch": 1024
}
}
}
src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs
+31 -13
View File
@@ -177,9 +177,10 @@ public sealed class GatewayOptionsValidatorTests
}
/// <summary>
/// A poll cadence below the 100 ms floor must fail validation. Both
/// values are stamped onto every worker launch environment, so they
/// are validated whether or not the central alarm monitor is enabled.
/// A poll cadence outside the 100 ms – 1 h range must fail validation.
/// Both values are stamped onto every worker launch environment, so
/// they are validated whether or not the central alarm monitor is
/// enabled.
/// </summary>
/// <param name="pollIntervalMilliseconds">Cadence under test.</param>
/// <param name="alarmsEnabled">Whether the central alarm monitor is on.</param>
@@ -188,7 +189,12 @@ public sealed class GatewayOptionsValidatorTests
[InlineData(0, false)]
[InlineData(-1, false)]
[InlineData(99, true)]
public void Validate_Fails_WhenAlarmPollIntervalBelowFloor(
// Above the one-hour ceiling the cadence stops being a cadence: int.MaxValue
// milliseconds is ~24 days, which silently disables alarm polling.
[InlineData(3_600_001, false)]
[InlineData(int.MaxValue, false)]
[InlineData(int.MaxValue, true)]
public void Validate_Fails_WhenAlarmPollIntervalOutOfRange(
int pollIntervalMilliseconds,
bool alarmsEnabled)
{
@@ -210,9 +216,12 @@ public sealed class GatewayOptionsValidatorTests
}
/// <summary>
/// A per-fetch cap below the 64-record floor must fail validation. The
/// cap doubles as the truncation-detection threshold in the worker, so
/// a tiny cap would make almost every fetch read as truncated.
/// A per-fetch cap outside the 64 – 65,536 range must fail validation.
/// The cap doubles as the truncation-detection threshold in the worker,
/// so a tiny cap would make almost every fetch read as truncated; and
/// the worker is a 32-bit process that materializes each reply as one
/// BSTR plus a full XmlDocument, so an unbounded cap is an
/// out-of-memory fault on the STA rather than a slow poll.
/// </summary>
/// <param name="maxAlarmsPerFetch">Cap under test.</param>
/// <param name="alarmsEnabled">Whether the central alarm monitor is on.</param>
@@ -221,7 +230,10 @@ public sealed class GatewayOptionsValidatorTests
[InlineData(0, false)]
[InlineData(-1, false)]
[InlineData(63, true)]
public void Validate_Fails_WhenMaxAlarmsPerFetchBelowFloor(
[InlineData(65_537, false)]
[InlineData(int.MaxValue, false)]
[InlineData(int.MaxValue, true)]
public void Validate_Fails_WhenMaxAlarmsPerFetchOutOfRange(
int maxAlarmsPerFetch,
bool alarmsEnabled)
{
@@ -242,9 +254,15 @@ public sealed class GatewayOptionsValidatorTests
f => f.Contains("MxGateway:Alarms:MaxAlarmsPerFetch", StringComparison.Ordinal));
}
/// <summary>Verifies the floor values themselves are accepted.</summary>
[Fact]
public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapFloors()
/// <summary>Verifies the boundary values themselves are accepted at both ends.</summary>
/// <param name="pollIntervalMilliseconds">Cadence under test.</param>
/// <param name="maxAlarmsPerFetch">Cap under test.</param>
[Theory]
[InlineData(100, 64)] // floors
[InlineData(3_600_000, 65_536)] // ceilings
public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapBoundaries(
int pollIntervalMilliseconds,
int maxAlarmsPerFetch)
{
GatewayOptions options = CloneWithAlarms(
ValidOptions(),
@@ -252,8 +270,8 @@ public sealed class GatewayOptionsValidatorTests
{
Enabled = true,
DefaultArea = "Galaxy",
PollIntervalMilliseconds = 100,
MaxAlarmsPerFetch = 64,
PollIntervalMilliseconds = pollIntervalMilliseconds,
MaxAlarmsPerFetch = maxAlarmsPerFetch,
});
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options);
src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs
+45
View File
@@ -57,6 +57,51 @@ public sealed class MxAccessStaSessionTests
}
}
/// <summary>
/// The alarm poll cadence comes from the launcher-set environment
/// variable; a missing, unparseable, or out-of-range value must fall
/// back to the 500 ms default rather than throw. The ceiling matters as
/// much as the floor: <see cref="int.MaxValue"/> milliseconds is ~24
/// days, which silently disables alarm polling altogether.
/// </summary>
/// <param name="environmentValue">Raw environment value under test.</param>
/// <param name="expectedMilliseconds">Expected resolved cadence, in milliseconds.</param>
[Theory]
[InlineData(null, 500)]
[InlineData("", 500)]
[InlineData("not-a-number", 500)]
[InlineData("0", 500)]
[InlineData("-1", 500)]
[InlineData("99", 500)]
[InlineData("100", 100)]
[InlineData("250", 250)]
[InlineData("3600000", 3600000)]
[InlineData("3600001", 500)]
[InlineData("2147483647", 500)]
public void ResolveAlarmPollInterval_WithEnvironmentValue_FallsBackToDefaultWhenOutOfRange(
string? environmentValue,
int expectedMilliseconds)
{
string? original = Environment.GetEnvironmentVariable(
MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName);
try
{
Environment.SetEnvironmentVariable(
MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName,
environmentValue);
Assert.Equal(
TimeSpan.FromMilliseconds(expectedMilliseconds),
MxAccessStaSession.ResolveAlarmPollInterval());
}
finally
{
Environment.SetEnvironmentVariable(
MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName,
original);
}
}
/// <summary>
/// Verifies that StartAsync creates the MXAccess COM object and attaches the event sink on the STA thread.
/// </summary>
src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs
+31
View File
@@ -496,6 +496,11 @@ public sealed class WnWrapAlarmConsumerXmlTests
[InlineData("63", 1024)]
[InlineData("64", 64)]
[InlineData("4096", 4096)]
[InlineData("65536", 65536)]
// Above the ceiling: the x86 worker materializes the whole reply as one
// BSTR plus an XmlDocument, so an unbounded cap is an OOM on the STA.
[InlineData("65537", 1024)]
[InlineData("2147483647", 1024)]
public void ResolveMaxAlarmsPerFetch_WithEnvironmentValue_FallsBackToDefaultWhenUnusable(
string? environmentValue,
int expected)
@@ -518,6 +523,32 @@ public sealed class WnWrapAlarmConsumerXmlTests
}
}
/// <summary>
/// A galaxy parked above the cap truncates on every poll, so the
/// warning must be throttled: two truncated polls inside one interval
/// produce exactly one line, and the next one only after the full
/// interval has elapsed.
/// </summary>
[Fact]
public void ShouldWarnTruncation_ThrottlesConsecutiveTruncatedPollsToOneWarningPerInterval()
{
// Seeded so the very first truncated poll always warns.
const long NeverWarned = -60_000;
// Poll 1 at t=0: warns, and records t=0 as the last warning.
Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(0, NeverWarned));
// Poll 2 half a second later (the default cadence): suppressed.
Assert.False(WnWrapAlarmConsumer.ShouldWarnTruncation(500, 0));
// Still suppressed just shy of the interval...
Assert.False(WnWrapAlarmConsumer.ShouldWarnTruncation(59_999, 0));
// ...and allowed again exactly on it.
Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(60_000, 0));
Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(120_000, 60_000));
}
/// <summary>
/// Builds a well-formed ALARM_RECORDS payload with
/// <paramref name="count"/> distinct alarms. GUIDs are the dashless
src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs
+12 -1
View File
@@ -35,6 +35,15 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
/// </summary>
internal static readonly TimeSpan MinimumAlarmPollInterval = TimeSpan.FromMilliseconds(100);
/// <summary>
/// Ceiling on the resolved alarm poll cadence (one hour). Mirrors the
/// gateway-side MxGateway:Alarms:PollIntervalMilliseconds maximum.
/// Without it, int.MaxValue milliseconds — which passed startup
/// validation before this existed — silently disables alarm polling
/// for ~24 days rather than reporting a misconfiguration.
/// </summary>
internal static readonly TimeSpan MaximumAlarmPollInterval = TimeSpan.FromMilliseconds(3_600_000);
/// <summary>Default alarm poll cadence when the environment says nothing usable.</summary>
internal static readonly TimeSpan DefaultAlarmPollInterval = TimeSpan.FromMilliseconds(500);
@@ -235,7 +244,9 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
}
TimeSpan resolved = TimeSpan.FromMilliseconds(milliseconds);
return resolved < MinimumAlarmPollInterval ? DefaultAlarmPollInterval : resolved;
return resolved < MinimumAlarmPollInterval || resolved > MaximumAlarmPollInterval
? DefaultAlarmPollInterval
: resolved;
}
/// <summary>
src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs
+55 -4
View File
@@ -58,10 +58,20 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
/// </summary>
internal const int MinimumMaxAlarmsPerFetch = 64;
/// <summary>
/// Ceiling on the resolved per-fetch cap. Mirrors the gateway-side
/// <c>MxGateway:Alarms:MaxAlarmsPerFetch</c> maximum. The worker is a
/// 32-bit process: every poll materializes the whole reply as one BSTR
/// and then a full <see cref="XmlDocument"/> over it, so an unbounded
/// cap (<see cref="int.MaxValue"/> passed startup validation before
/// this existed) is an out-of-memory fault on the STA, not a slow poll.
/// </summary>
internal const int MaximumMaxAlarmsPerFetch = 65_536;
/// <summary>
/// Environment variable the gateway's <c>WorkerProcessLauncher</c>
/// sets from <c>MxGateway:Alarms:MaxAlarmsPerFetch</c>. A missing,
/// unparseable, or below-floor value falls back to
/// unparseable, or out-of-range value falls back to
/// <see cref="DefaultMaxAlarmsPerFetch"/> — a bad environment value
/// must never keep the alarm consumer from starting.
/// </summary>
@@ -117,10 +127,13 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
/// <summary>
/// Resolves the per-fetch cap from the launcher-provided environment
/// variable. A missing, unparseable, or below-floor value falls back
/// variable. A missing, unparseable, or out-of-range value falls back
/// to <see cref="DefaultMaxAlarmsPerFetch"/> rather than throwing:
/// polling at the default cap is always safe, and a session that
/// refuses to subscribe over a mistyped environment variable is not.
/// The ceiling is enforced here as well as in the gateway validator so
/// a hand-set environment (or a future launcher bug) cannot hand the
/// x86 worker a cap that faults it on the first poll.
/// </summary>
/// <returns>The cap passed to <c>GetXmlCurrentAlarms2</c>.</returns>
internal static int ResolveMaxAlarmsPerFetch()
@@ -132,6 +145,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
CultureInfo.InvariantCulture,
out int cap)
&& cap >= MinimumMaxAlarmsPerFetch
&& cap <= MaximumMaxAlarmsPerFetch
? cap
: DefaultMaxAlarmsPerFetch;
}
@@ -498,7 +512,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
{
total = ++truncatedFetchCount;
elapsed = truncationWarningClock.ElapsedMilliseconds;
if (elapsed - lastTruncationWarningMilliseconds < TruncationWarningIntervalMilliseconds)
if (!ShouldWarnTruncation(elapsed, lastTruncationWarningMilliseconds))
{
return;
}
@@ -506,13 +520,24 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
lastTruncationWarningMilliseconds = elapsed;
}
// Format matches WorkerConsoleLogger's "level=<L> event=<E> k=v k=v"
// line shape so the worker's stderr parses uniformly. Deviation, on
// purpose: IWorkerLogger exposes only Information and Error, so
// "Warning" is a third level string no other worker line emits. A
// truncated snapshot is not an error (the poll succeeded and the
// snapshot is safe) but it is not routine either, so downgrading it to
// Information would bury it. Revisit if the truncation signal becomes
// structural — see docs/DesignDecisions.md.
//
// Identifiers and counts only — no tag names, values, limits, or
// comments, per the gateway's "don't log tag values by default" rule.
// The trailing note stays a single bare k=v token (no spaces, no
// semicolons); remediation prose lives in docs/GatewayConfiguration.md.
string message = string.Format(
CultureInfo.InvariantCulture,
"level=Warning event=AlarmSnapshotTruncated maxAlarmsPerFetch={0} fetchedRecords={1} "
+ "parsedRecords={2} retainedSnapshotSize={3} truncatedFetchesSinceStart={4} "
+ "note=absence-implies-clear suppressed for this poll; raise MxGateway:Alarms:MaxAlarmsPerFetch",
+ "note=truncated-snapshot-retained",
maxAlarmsPerFetch,
fetchedRecordCount,
parsedRecordCount,
@@ -523,6 +548,25 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
sink(message);
}
/// <summary>
/// Rate-limit decision for the truncated-fetch warning: emit only when
/// a full <see cref="TruncationWarningIntervalMilliseconds"/> has
/// elapsed since the last one. A galaxy parked above the cap truncates
/// on every poll, so consecutive truncated polls inside one interval
/// must produce exactly one line. Exposed as <c>internal static</c>
/// so the throttle is unit-testable without the wnwrapConsumer COM
/// object.
/// </summary>
/// <param name="elapsedMilliseconds">Consumer-lifetime clock reading for this poll.</param>
/// <param name="lastWarningMilliseconds">Clock reading when the last warning was emitted.</param>
/// <returns><see langword="true"/> when this poll may emit a warning.</returns>
internal static bool ShouldWarnTruncation(
long elapsedMilliseconds,
long lastWarningMilliseconds)
{
return elapsedMilliseconds - lastWarningMilliseconds >= TruncationWarningIntervalMilliseconds;
}
/// <summary>
/// Pure snapshot-to-transitions diff. Compares the previous polled
/// snapshot to the next snapshot and produces one
@@ -641,6 +685,13 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer
// first-match-wins (null means "not seen yet"), and a single
// coalesce to string.Empty at the end — which keeps the absent and
// empty cases indistinguishable exactly as before.
//
// One theoretical divergence: XmlNode.Name is the QName, so a child
// carrying its own default-namespace declaration (<GUID xmlns="...">)
// matches here where the unprefixed XPath name test would not. It is
// theoretical — wnwrap emits no namespaces, and a namespaced ALARM
// wrapper makes the outer SelectNodes return nothing either way — and
// it errs permissive (field populated rather than silently dropped).
string? guidHex = null;
string? xmlDate = null;
string? xmlTime = null;