From b5ea6bb461eff05f3d5e109b32edbb7dc5effcfd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:19:41 -0400 Subject: [PATCH] fix(alarms): fetch/poll ceilings; truncation-semantics docs; log-format conformance --- docs/AlarmClientDiscovery.md | 20 +++++++ docs/DesignDecisions.md | 28 +++++++-- docs/GatewayConfiguration.md | 4 +- .../Configuration/AlarmsOptions.cs | 26 ++++---- .../Configuration/GatewayOptionsValidator.cs | 20 +++++-- .../appsettings.json | 4 +- .../GatewayOptionsValidatorTests.cs | 44 ++++++++++---- .../MxAccess/MxAccessStaSessionTests.cs | 45 ++++++++++++++ .../MxAccess/WnWrapAlarmConsumerXmlTests.cs | 31 ++++++++++ .../MxAccess/MxAccessStaSession.cs | 13 +++- .../MxAccess/WnWrapAlarmConsumer.cs | 59 +++++++++++++++++-- 11 files changed, 254 insertions(+), 40 deletions(-) diff --git a/docs/AlarmClientDiscovery.md b/docs/AlarmClientDiscovery.md index e03ac9d..f43fff8 100644 --- a/docs/AlarmClientDiscovery.md +++ b/docs/AlarmClientDiscovery.md @@ -140,6 +140,26 @@ Two viable A.2 designs given the probe data: poll period; modest CPU floor because the call is cheap. Matches the heartbeat-style WM 0xC275 semantics — AVEVA itself runs a poll loop internally. + + As shipped, this is the chosen design, and the cadence is **no + longer fixed at 500 ms**: it is the 500 ms *default* of + `MxGateway:Alarms:PollIntervalMilliseconds` (range 100 ms – 1 h), + which the gateway hands the worker through the + `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable. The + per-fetch cap is likewise configurable + (`MxGateway:Alarms:MaxAlarmsPerFetch`, default 1024). + + One snapshot rule matters when reading the capture below: a fetch + that returns exactly the cap is treated as **truncated**, and the + worker *merges* it into the retained snapshot instead of replacing + it. `GetXmlCurrentAlarms2` caps its reply with no "more available" + flag, so a capped reply is authoritative about presence only — + alarms it had no room to mention are retained rather than allowed + to vanish, because their disappearance is what the gateway's + reconcile pass reads as a clear. Only a sub-cap fetch replaces the + snapshot wholesale and can therefore clear alarms. See + `docs/DesignDecisions.md`, "Alarms — a capped snapshot fetch never + implies a clear". 2. **Hook AVEVA's internal window.** Discover AVEVA's own window (`hwnd=0x18032E` in the probe), `SetWindowsHookEx` or `SetWindowSubclass` on it, and intercept WM 0xC275 on AVEVA's diff --git a/docs/DesignDecisions.md b/docs/DesignDecisions.md index 6a1d8f3..e315ce0 100644 --- a/docs/DesignDecisions.md +++ b/docs/DesignDecisions.md @@ -172,6 +172,19 @@ Consequences, and how this sits with the existing failover/reconcile design: happened is broadcast to every `StreamAlarms` subscriber and cannot be taken back. Consumers already apply transitions as "set this alarm to this state", so a repeated or delayed Clear is absorbed. +- **Under *sustained* truncation, some intermediate history is lost — end state + is not.** For an alarm that stays outside the fetch window, a full + clear→re-raise cycle that begins and ends between two sightings emits **no + transitions at all**: the retained record is identical before and after, so + the diff sees nothing to report. Consumers that render current state are + correct; consumers that *count occurrences* lose an event. Likewise, an + operator acknowledgement of an out-of-window alarm does not reach the feed + until that alarm re-enters a fetch window, at which point the reconcile + repairs the acked state. This is a strictly better failure than the + pre-guard behaviour (which fabricated a Clear for every out-of-window alarm + on every poll), but it is not lossless, and it is another reason a + persistently truncating deployment is a configuration defect to fix rather + than a mode to run in. - **It does not synthesize anything.** Suppressing an inference is the opposite of inventing an event; no transition is fabricated on a truncated poll. - **Failover is unaffected.** `FailoverAlarmConsumer` selects which @@ -179,10 +192,17 @@ Consequences, and how this sits with the existing failover/reconcile design: consumer's own snapshot bookkeeping and changes neither the failure counting that triggers failover nor the subtag standby's snapshot, which is built from a bounded watch-list and has no per-fetch cap to hit. -- **Operators get told.** A truncated poll logs a rate-limited (once per - minute) `AlarmSnapshotTruncated` warning carrying the cap, the record counts, - and the running truncated-fetch total — identifiers and counts only, never - tag names, values, limits, or comments. A galaxy that truncates persistently +- **Operators get told, weakly.** A truncated poll logs a rate-limited (once + per minute) `AlarmSnapshotTruncated` warning carrying the cap, the record + counts, and the running truncated-fetch total — identifiers and counts only, + never tag names, values, limits, or comments. Be honest about its reach: it + goes to the worker's console/stderr, which is captured on dev hosts but is + not a metric, not a dashboard tile, and not part of any session-status or + alarm-feed payload, so a production deployment can truncate indefinitely + without anyone noticing. Surfacing truncation as a **structural** degraded + status (a field on the alarm-provider mode/status surface the dashboard and + `StreamAlarms` consumers already read) is filed as a follow-up; until it + lands, the log line is the only signal. A galaxy that truncates persistently is a configuration problem: raise `MxGateway:Alarms:MaxAlarmsPerFetch`. ## Session-Resilience Epic Scope diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 7d73c9e..5feb37a 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -418,8 +418,8 @@ behavior. | `MxGateway:Alarms:SubscriptionExpression` | _(empty)_ | AVEVA alarm-subscription expression the monitor subscribes on startup, in canonical `\\\Galaxy!` form. The literal `Galaxy` provider is correct regardless of the Galaxy database name. When empty and `Enabled` is `true`, the gateway falls back to `\\\Galaxy!` if `DefaultArea` is set. | | `MxGateway:Alarms:DefaultArea` | _(empty)_ | Area name used to compose a default subscription when `SubscriptionExpression` is empty. If both are empty while `Enabled` is `true`, the monitor faults with a configuration diagnostic. | | `MxGateway:Alarms:ReconcileIntervalSeconds` | `30` | How often the monitor reconciles its in-process alarm cache against the worker's authoritative active-alarm snapshot, catching transitions the live poll-and-diff feed missed. Floored at 5 seconds. | -| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be `>= 100`: every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or unusable value leaves the worker on the 500 ms default rather than failing the session. | -| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be `>= 64`. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning (identifiers and counts only). Raise this on galaxies whose steady-state active-alarm count approaches the cap — a galaxy permanently above it holds stale entries in the snapshot until a sub-cap poll. Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or unusable value leaves the worker on the 1024 default. | +| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be between `100` and `3600000` (one hour): every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path, while a value above an hour stops being a cadence and silently disables alarm polling. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or out-of-range value leaves the worker on the 500 ms default rather than failing the session. | +| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be between `64` and `65536` — the worker is a 32-bit process that materializes each reply as one BSTR plus a full `XmlDocument`, so an unbounded cap faults the STA with an out-of-memory rather than merely slowing it. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning to its stderr (identifiers and counts only). **Remediation when you see that warning: raise this value** so the steady-state active-alarm count fits inside one fetch. A galaxy permanently above the cap holds stale entries in the snapshot until a sub-cap poll, and loses clear→re-raise cycles that happen entirely out of window (see `docs/DesignDecisions.md`). Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or out-of-range value leaves the worker on the 1024 default. | The alarm monitor is independent of client sessions: `AcknowledgeAlarm` and `StreamAlarms` are session-less RPCs served by the monitor. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs index 98259f5..94363bf 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs @@ -49,22 +49,26 @@ public sealed class AlarmsOptions /// /// Cadence at which the worker's STA polls the AVEVA alarm consumer /// (GetXmlCurrentAlarms2) for the current active-alarm snapshot. - /// Default 500 ms; must be at least 100 ms. Every poll is a COM call - /// plus an XML parse on the STA that also serves reads and writes, so - /// driving it below 100 ms starves the command path. Conveyed to the - /// worker through the MXGATEWAY_ALARM_POLL_INTERVAL_MS - /// environment variable. + /// Default 500 ms; must be between 100 ms and 3,600,000 ms (one hour). + /// Every poll is a COM call plus an XML parse on the STA that also + /// serves reads and writes, so driving it below 100 ms starves the + /// command path; above an hour the cadence stops being a cadence and + /// silently disables alarm polling. Conveyed to the worker through the + /// MXGATEWAY_ALARM_POLL_INTERVAL_MS environment variable. /// public int PollIntervalMilliseconds { get; init; } = 500; /// /// Cap the worker passes to GetXmlCurrentAlarms2's - /// maxAlmCnt argument. Default 1024; must be at least 64. A - /// fetch that comes back holding exactly this many records is treated - /// as truncated: the worker keeps the alarms the capped fetch could - /// not mention in its snapshot rather than letting their absence read - /// as a clear. Raise it on galaxies whose steady-state active-alarm - /// count approaches the cap. Conveyed to the worker through the + /// maxAlmCnt argument. Default 1024; must be between 64 and + /// 65,536 — the worker is a 32-bit process that materializes each + /// fetch as one BSTR plus a full XmlDocument, so an unbounded cap + /// faults the STA rather than merely slowing it. A fetch that comes + /// back holding exactly this many records is treated as truncated: the + /// worker keeps the alarms the capped fetch could not mention in its + /// snapshot rather than letting their absence read as a clear. Raise it + /// on galaxies whose steady-state active-alarm count approaches the + /// cap. Conveyed to the worker through the /// MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH environment variable. /// public int MaxAlarmsPerFetch { get; init; } = 1024; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 55e31c1..eadeea3 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -427,23 +427,35 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase MaximumAlarmPollIntervalMilliseconds) { builder.Add( - $"MxGateway:Alarms:PollIntervalMilliseconds must be greater than or equal to {MinimumAlarmPollIntervalMilliseconds}."); + $"MxGateway:Alarms:PollIntervalMilliseconds must be between {MinimumAlarmPollIntervalMilliseconds} and {MaximumAlarmPollIntervalMilliseconds}."); } - if (options.MaxAlarmsPerFetch < MinimumMaxAlarmsPerFetch) + if (options.MaxAlarmsPerFetch is < MinimumMaxAlarmsPerFetch or > MaximumMaxAlarmsPerFetch) { builder.Add( - $"MxGateway:Alarms:MaxAlarmsPerFetch must be greater than or equal to {MinimumMaxAlarmsPerFetch}."); + $"MxGateway:Alarms:MaxAlarmsPerFetch must be between {MinimumMaxAlarmsPerFetch} and {MaximumMaxAlarmsPerFetch}."); } if (!options.Enabled) diff --git a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json index e99f5e0..ad80bcf 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json +++ b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json @@ -86,7 +86,9 @@ "Enabled": true, "SubscriptionExpression": "\\\\DESKTOP-6JL3KKO\\Galaxy!DEV", "DefaultArea": "", - "ReconcileIntervalSeconds": 30 + "ReconcileIntervalSeconds": 30, + "PollIntervalMilliseconds": 500, + "MaxAlarmsPerFetch": 1024 } } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index a5cb35a..2878517 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -177,9 +177,10 @@ public sealed class GatewayOptionsValidatorTests } /// - /// A poll cadence below the 100 ms floor must fail validation. Both - /// values are stamped onto every worker launch environment, so they - /// are validated whether or not the central alarm monitor is enabled. + /// A poll cadence outside the 100 ms – 1 h range must fail validation. + /// Both values are stamped onto every worker launch environment, so + /// they are validated whether or not the central alarm monitor is + /// enabled. /// /// Cadence under test. /// Whether the central alarm monitor is on. @@ -188,7 +189,12 @@ public sealed class GatewayOptionsValidatorTests [InlineData(0, false)] [InlineData(-1, false)] [InlineData(99, true)] - public void Validate_Fails_WhenAlarmPollIntervalBelowFloor( + // Above the one-hour ceiling the cadence stops being a cadence: int.MaxValue + // milliseconds is ~24 days, which silently disables alarm polling. + [InlineData(3_600_001, false)] + [InlineData(int.MaxValue, false)] + [InlineData(int.MaxValue, true)] + public void Validate_Fails_WhenAlarmPollIntervalOutOfRange( int pollIntervalMilliseconds, bool alarmsEnabled) { @@ -210,9 +216,12 @@ public sealed class GatewayOptionsValidatorTests } /// - /// A per-fetch cap below the 64-record floor must fail validation. The - /// cap doubles as the truncation-detection threshold in the worker, so - /// a tiny cap would make almost every fetch read as truncated. + /// A per-fetch cap outside the 64 – 65,536 range must fail validation. + /// The cap doubles as the truncation-detection threshold in the worker, + /// so a tiny cap would make almost every fetch read as truncated; and + /// the worker is a 32-bit process that materializes each reply as one + /// BSTR plus a full XmlDocument, so an unbounded cap is an + /// out-of-memory fault on the STA rather than a slow poll. /// /// Cap under test. /// Whether the central alarm monitor is on. @@ -221,7 +230,10 @@ public sealed class GatewayOptionsValidatorTests [InlineData(0, false)] [InlineData(-1, false)] [InlineData(63, true)] - public void Validate_Fails_WhenMaxAlarmsPerFetchBelowFloor( + [InlineData(65_537, false)] + [InlineData(int.MaxValue, false)] + [InlineData(int.MaxValue, true)] + public void Validate_Fails_WhenMaxAlarmsPerFetchOutOfRange( int maxAlarmsPerFetch, bool alarmsEnabled) { @@ -242,9 +254,15 @@ public sealed class GatewayOptionsValidatorTests f => f.Contains("MxGateway:Alarms:MaxAlarmsPerFetch", StringComparison.Ordinal)); } - /// Verifies the floor values themselves are accepted. - [Fact] - public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapFloors() + /// Verifies the boundary values themselves are accepted at both ends. + /// Cadence under test. + /// Cap under test. + [Theory] + [InlineData(100, 64)] // floors + [InlineData(3_600_000, 65_536)] // ceilings + public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapBoundaries( + int pollIntervalMilliseconds, + int maxAlarmsPerFetch) { GatewayOptions options = CloneWithAlarms( ValidOptions(), @@ -252,8 +270,8 @@ public sealed class GatewayOptionsValidatorTests { Enabled = true, DefaultArea = "Galaxy", - PollIntervalMilliseconds = 100, - MaxAlarmsPerFetch = 64, + PollIntervalMilliseconds = pollIntervalMilliseconds, + MaxAlarmsPerFetch = maxAlarmsPerFetch, }); ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs index 44894ea..aace1d7 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs @@ -57,6 +57,51 @@ public sealed class MxAccessStaSessionTests } } + /// + /// The alarm poll cadence comes from the launcher-set environment + /// variable; a missing, unparseable, or out-of-range value must fall + /// back to the 500 ms default rather than throw. The ceiling matters as + /// much as the floor: milliseconds is ~24 + /// days, which silently disables alarm polling altogether. + /// + /// Raw environment value under test. + /// Expected resolved cadence, in milliseconds. + [Theory] + [InlineData(null, 500)] + [InlineData("", 500)] + [InlineData("not-a-number", 500)] + [InlineData("0", 500)] + [InlineData("-1", 500)] + [InlineData("99", 500)] + [InlineData("100", 100)] + [InlineData("250", 250)] + [InlineData("3600000", 3600000)] + [InlineData("3600001", 500)] + [InlineData("2147483647", 500)] + public void ResolveAlarmPollInterval_WithEnvironmentValue_FallsBackToDefaultWhenOutOfRange( + string? environmentValue, + int expectedMilliseconds) + { + string? original = Environment.GetEnvironmentVariable( + MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName); + try + { + Environment.SetEnvironmentVariable( + MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName, + environmentValue); + + Assert.Equal( + TimeSpan.FromMilliseconds(expectedMilliseconds), + MxAccessStaSession.ResolveAlarmPollInterval()); + } + finally + { + Environment.SetEnvironmentVariable( + MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName, + original); + } + } + /// /// Verifies that StartAsync creates the MXAccess COM object and attaches the event sink on the STA thread. /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs index 682365d..7dd9448 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs @@ -496,6 +496,11 @@ public sealed class WnWrapAlarmConsumerXmlTests [InlineData("63", 1024)] [InlineData("64", 64)] [InlineData("4096", 4096)] + [InlineData("65536", 65536)] + // Above the ceiling: the x86 worker materializes the whole reply as one + // BSTR plus an XmlDocument, so an unbounded cap is an OOM on the STA. + [InlineData("65537", 1024)] + [InlineData("2147483647", 1024)] public void ResolveMaxAlarmsPerFetch_WithEnvironmentValue_FallsBackToDefaultWhenUnusable( string? environmentValue, int expected) @@ -518,6 +523,32 @@ public sealed class WnWrapAlarmConsumerXmlTests } } + /// + /// A galaxy parked above the cap truncates on every poll, so the + /// warning must be throttled: two truncated polls inside one interval + /// produce exactly one line, and the next one only after the full + /// interval has elapsed. + /// + [Fact] + public void ShouldWarnTruncation_ThrottlesConsecutiveTruncatedPollsToOneWarningPerInterval() + { + // Seeded so the very first truncated poll always warns. + const long NeverWarned = -60_000; + + // Poll 1 at t=0: warns, and records t=0 as the last warning. + Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(0, NeverWarned)); + + // Poll 2 half a second later (the default cadence): suppressed. + Assert.False(WnWrapAlarmConsumer.ShouldWarnTruncation(500, 0)); + + // Still suppressed just shy of the interval... + Assert.False(WnWrapAlarmConsumer.ShouldWarnTruncation(59_999, 0)); + + // ...and allowed again exactly on it. + Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(60_000, 0)); + Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(120_000, 60_000)); + } + /// /// Builds a well-formed ALARM_RECORDS payload with /// distinct alarms. GUIDs are the dashless diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs index 6c2a480..6050993 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs @@ -35,6 +35,15 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession /// internal static readonly TimeSpan MinimumAlarmPollInterval = TimeSpan.FromMilliseconds(100); + /// + /// Ceiling on the resolved alarm poll cadence (one hour). Mirrors the + /// gateway-side MxGateway:Alarms:PollIntervalMilliseconds maximum. + /// Without it, int.MaxValue milliseconds — which passed startup + /// validation before this existed — silently disables alarm polling + /// for ~24 days rather than reporting a misconfiguration. + /// + internal static readonly TimeSpan MaximumAlarmPollInterval = TimeSpan.FromMilliseconds(3_600_000); + /// Default alarm poll cadence when the environment says nothing usable. internal static readonly TimeSpan DefaultAlarmPollInterval = TimeSpan.FromMilliseconds(500); @@ -235,7 +244,9 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession } TimeSpan resolved = TimeSpan.FromMilliseconds(milliseconds); - return resolved < MinimumAlarmPollInterval ? DefaultAlarmPollInterval : resolved; + return resolved < MinimumAlarmPollInterval || resolved > MaximumAlarmPollInterval + ? DefaultAlarmPollInterval + : resolved; } /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs index 18085e1..b9f44f4 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs @@ -58,10 +58,20 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// internal const int MinimumMaxAlarmsPerFetch = 64; + /// + /// Ceiling on the resolved per-fetch cap. Mirrors the gateway-side + /// MxGateway:Alarms:MaxAlarmsPerFetch maximum. The worker is a + /// 32-bit process: every poll materializes the whole reply as one BSTR + /// and then a full over it, so an unbounded + /// cap ( passed startup validation before + /// this existed) is an out-of-memory fault on the STA, not a slow poll. + /// + internal const int MaximumMaxAlarmsPerFetch = 65_536; + /// /// Environment variable the gateway's WorkerProcessLauncher /// sets from MxGateway:Alarms:MaxAlarmsPerFetch. A missing, - /// unparseable, or below-floor value falls back to + /// unparseable, or out-of-range value falls back to /// — a bad environment value /// must never keep the alarm consumer from starting. /// @@ -117,10 +127,13 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// /// Resolves the per-fetch cap from the launcher-provided environment - /// variable. A missing, unparseable, or below-floor value falls back + /// variable. A missing, unparseable, or out-of-range value falls back /// to rather than throwing: /// polling at the default cap is always safe, and a session that /// refuses to subscribe over a mistyped environment variable is not. + /// The ceiling is enforced here as well as in the gateway validator so + /// a hand-set environment (or a future launcher bug) cannot hand the + /// x86 worker a cap that faults it on the first poll. /// /// The cap passed to GetXmlCurrentAlarms2. internal static int ResolveMaxAlarmsPerFetch() @@ -132,6 +145,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer CultureInfo.InvariantCulture, out int cap) && cap >= MinimumMaxAlarmsPerFetch + && cap <= MaximumMaxAlarmsPerFetch ? cap : DefaultMaxAlarmsPerFetch; } @@ -498,7 +512,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer { total = ++truncatedFetchCount; elapsed = truncationWarningClock.ElapsedMilliseconds; - if (elapsed - lastTruncationWarningMilliseconds < TruncationWarningIntervalMilliseconds) + if (!ShouldWarnTruncation(elapsed, lastTruncationWarningMilliseconds)) { return; } @@ -506,13 +520,24 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer lastTruncationWarningMilliseconds = elapsed; } + // Format matches WorkerConsoleLogger's "level= event= k=v k=v" + // line shape so the worker's stderr parses uniformly. Deviation, on + // purpose: IWorkerLogger exposes only Information and Error, so + // "Warning" is a third level string no other worker line emits. A + // truncated snapshot is not an error (the poll succeeded and the + // snapshot is safe) but it is not routine either, so downgrading it to + // Information would bury it. Revisit if the truncation signal becomes + // structural — see docs/DesignDecisions.md. + // // Identifiers and counts only — no tag names, values, limits, or // comments, per the gateway's "don't log tag values by default" rule. + // The trailing note stays a single bare k=v token (no spaces, no + // semicolons); remediation prose lives in docs/GatewayConfiguration.md. string message = string.Format( CultureInfo.InvariantCulture, "level=Warning event=AlarmSnapshotTruncated maxAlarmsPerFetch={0} fetchedRecords={1} " + "parsedRecords={2} retainedSnapshotSize={3} truncatedFetchesSinceStart={4} " - + "note=absence-implies-clear suppressed for this poll; raise MxGateway:Alarms:MaxAlarmsPerFetch", + + "note=truncated-snapshot-retained", maxAlarmsPerFetch, fetchedRecordCount, parsedRecordCount, @@ -523,6 +548,25 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer sink(message); } + /// + /// Rate-limit decision for the truncated-fetch warning: emit only when + /// a full has + /// elapsed since the last one. A galaxy parked above the cap truncates + /// on every poll, so consecutive truncated polls inside one interval + /// must produce exactly one line. Exposed as internal static + /// so the throttle is unit-testable without the wnwrapConsumer COM + /// object. + /// + /// Consumer-lifetime clock reading for this poll. + /// Clock reading when the last warning was emitted. + /// when this poll may emit a warning. + internal static bool ShouldWarnTruncation( + long elapsedMilliseconds, + long lastWarningMilliseconds) + { + return elapsedMilliseconds - lastWarningMilliseconds >= TruncationWarningIntervalMilliseconds; + } + /// /// Pure snapshot-to-transitions diff. Compares the previous polled /// snapshot to the next snapshot and produces one @@ -641,6 +685,13 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer // first-match-wins (null means "not seen yet"), and a single // coalesce to string.Empty at the end — which keeps the absent and // empty cases indistinguishable exactly as before. + // + // One theoretical divergence: XmlNode.Name is the QName, so a child + // carrying its own default-namespace declaration () + // matches here where the unprefixed XPath name test would not. It is + // theoretical — wnwrap emits no namespaces, and a namespaced ALARM + // wrapper makes the outer SelectNodes return nothing either way — and + // it errs permissive (field populated rather than silently dropped). string? guidHex = null; string? xmlDate = null; string? xmlTime = null;