From bdbc47f5fcf244cfea6f838d0897b64f91ccbb3c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 11:56:30 -0500 Subject: [PATCH 01/28] fix(mtconnect): serve an empty string-typed value as Good "" (#824) Owner decision 2026-11-11 (Java parity): a genuinely empty observation on a String/Reference or unauthored item is Good "" instead of BadNoCommunication. Non-string items (numeric, bool, DateTime, arrays) keep BadNoCommunication - the Agent supplied no value; BadTypeMismatch would wrongly blame the authored type. The UNAVAILABLE sentinel is unchanged, and a structured observation with no text stays BadNotSupported (structured check now precedes the empty check). Parity matrix, quality-mapping section and drivers rule updated. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .claude/rules/drivers.md | 5 + docs/drivers/MTConnect.md | 18 +++- .../MTConnectObservationIndex.cs | 31 ++++-- .../MTConnectObservationIndexTests.cs | 96 ++++++++++++++++++- 4 files changed, 136 insertions(+), 14 deletions(-) diff --git a/.claude/rules/drivers.md b/.claude/rules/drivers.md index abe6ffff9..8cfe9688a 100644 --- a/.claude/rules/drivers.md +++ b/.claude/rules/drivers.md @@ -312,6 +312,11 @@ host logger factory (wired in `DriverFactoryBootstrap`; the FOCAS and TwinCAT fa the adaptive raised count is clamped to the known `bufferSize`, #846). Every parse prohibits DTDs; the streams walk is depth-bounded (64 — an unbounded recursion is a process-killing `StackOverflowException`). Non-finite floats are never Good; integer-valued decimals serve integral tags losslessly. Matrix: `docs/drivers/MTConnect.md`. +- **MTConnect empty value (#824, Java parity):** an empty/whitespace observation on a string-typed (or unauthored) item is + Good `""`; on a non-string item (numeric/bool/DateTime/array) it stays `BadNoCommunication` (no value — not + `BadTypeMismatch`). Only the literal `UNAVAILABLE` is the no-value sentinel on a string item; it, the comms-loss + fan-out and the condition path are unchanged. A ScadaBridge string tag that used to go Bad on a blank value now + receives Good `""`. - **MTConnect refused-connection fast path (#846):** a `/sample`/re-baseline failure whose chain holds `SocketError.ConnectionRefused`, unbroken for `CommsLossConfirmMs` (default 2000; 0 = first; negative = off), publishes every subscribed value `BadNoCommunication` once per outage; the first delivery afterwards republishes all of them from diff --git a/docs/drivers/MTConnect.md b/docs/drivers/MTConnect.md index 1b84a9bee..d79624288 100644 --- a/docs/drivers/MTConnect.md +++ b/docs/drivers/MTConnect.md @@ -271,6 +271,18 @@ item. A `CONDITION` observation's value is taken from its **element name** condition element normalises onto the same `UNAVAILABLE` sentinel as every other category so one comparison covers all three. +**An empty value is not `UNAVAILABLE` (#824, owner decision 2026-11-11 — Java parity).** A +genuinely empty observation (``, ``, or whitespace-only text, +which the parser trims to `""`) on a **string-typed** item (`String`/`Reference`, or an unauthored +item, which coerces as String) is served **Good `""`** — an Agent can legitimately report a blank +`Program`, `Block` or `Message`. On any **non-string** item (numeric, `Boolean`, `DateTime`, a +`TIME_SERIES` array) `""` cannot be a value, so it stays **`BadNoCommunication`** with a null value +— deliberately not `BadTypeMismatch`, which would tell the operator the Agent sent text of the wrong +type and point them at the tag's authored data type; the Agent sent no value at all. Only the literal +`UNAVAILABLE` sentinel is the "no value" statement on a string item. Before #824 every empty value +was `BadNoCommunication`. A `DATA_SET`/`TABLE`-shaped observation is unaffected (its entry-less +form is still Good `{}`; non-`` children are still `BadNotSupported`). + **Integer-valued decimals are served on integral tags (#804).** Agents format integer-valued items as `42.000000`; an `Int*`/`UInt*` tag accepts any decimal whose value *is* an integer in range (`42.000000` → `42`, `1.5e3` → `1500`) losslessly. A real fraction stays `BadOutOfRange` — deliberately stricter than the Java @@ -296,7 +308,7 @@ limitations 4; a probe that does declare one yields a fixed `ArrayDim`). Parsing: the observation text is split on any whitespace, each token parsed invariant-culture. The result is **all-or-nothing** — one non-numeric / non-finite token is `BadTypeMismatch`, a literal `UNAVAILABLE` token or an -empty body is `BadNoCommunication`, and a count that disagrees with the +empty body is `BadNoCommunication` (an array is non-string, so #824's Good `""` never applies), and a count that disagrees with the observation's own `sampleCount` attribute (or a fixed `ArrayDim`) is `BadDecodingError`. A partial vector is never served as Good. Only a **Float64** array tag is served; any other declared element type stays @@ -677,7 +689,7 @@ deliberately different, kept. | Type inference SAMPLE→Double, numeric EVENT→Int64, else String (`MTConnectTypes`) | covered, **.NET exceeds** (TIME_SERIES → `double[]`, separator-insensitive types) | `MTConnectDataTypeInference` · `MTConnectDataTypeInferenceTests` | | UNAVAILABLE → `Bad_NoCommunication`, null value, source timestamp kept (`MTConnectValues`) | covered | `MTConnectObservationIndex.BuildSnapshot` · `Unavailable_observation_maps_to_BadNoCommunication_with_null_value`, `Unavailable_observation_keeps_the_agent_source_timestamp` | | UNAVAILABLE matched case-insensitively | divergence (kept) | ordinal match on the parser-normalised token, so a free-text EVENT reading "Unavailable" stays a value · `Unavailable_sentinel_is_matched_on_the_exact_literal` | -| Blank text on a String node is a Good empty string | divergence (kept) | empty text is `BadNoCommunication` (an empty element is not an empty-string value) | +| Blank text on a String node is a Good empty string | **covered** (was a divergence until #824, owner decision 2026-11-11) | `BuildSnapshot` serves Good `""` on a string-typed or unauthored item; a non-string item stays `BadNoCommunication` (Java has no typed node to refuse `""` on); `UNAVAILABLE` unchanged · `Empty_observation_value_on_a_string_item_is_a_Good_empty_string`, `A_parsed_empty_event_on_a_string_item_is_a_Good_empty_string`, controls `Empty_observation_value_on_a_non_string_item_stays_BadNoCommunication`, `Unavailable_on_a_string_item_is_still_BadNoCommunication` | | No fabricated numbers: narrow-integer range checks, float overflow ≠ Infinity, NaN/Infinity refused (`MTConnectValues.coerce`) | covered for integers; **gap → closed** for floats (`1e400`, `NaN`, `Infinity`, `1e39` on Float32 were served Good) | `Coerce`, `NonFiniteStatus` · `A_non_finite_float_is_never_served_as_good` | | Integer-valued decimal (`42.000000`) on an integral node | **gap → closed** (was `BadOutOfRange`) — lossless only; a real fraction is still refused, never truncated (Java truncates) | `TryIntegral` · `An_integer_valued_decimal_converts_losslessly`, control `A_fraction_or_an_out_of_range_decimal_stays_out_of_range` | | Source timestamps preserved; a non-Zulu offset normalised | covered | `ReadTimestampUtc` · `A_timestamp_with_a_non_zulu_offset_is_normalized_to_the_same_utc_instant` | @@ -932,7 +944,7 @@ for anything beyond values-and-conditions. - **Unit tests** — `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/` (canned-XML fixtures, no network) — 714/714 at #804 (2026-11-10). `MTConnectJavaParityTests` holds the #804 gap tests and the ported Java regression cases (see [Java reference parity](#java-reference-parity-804)). Covers discovery tree shape, `MTConnectDataTypeInference`, observation indexing, - `UNAVAILABLE` → `BadNoCommunication`, CONDITION state-word mapping, + `UNAVAILABLE` → `BadNoCommunication` (an empty string-typed value is Good `""`, #824), CONDITION state-word mapping, multipart chunk framing, and ring-buffer re-baseline paging (both the sequence-gap and the `OUT_OF_RANGE` legs). The write path adds `MTConnectAgentClientWriteTests` (`9f17d7ef` — verb, URL, agent-error-in-200 diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectObservationIndex.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectObservationIndex.cs index 40f6d0ada..5e24da78f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectObservationIndex.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectObservationIndex.cs @@ -76,7 +76,10 @@ internal sealed class MTConnectObservationIndex private const uint Good = 0x00000000u; - /// The Agent reported UNAVAILABLE, or supplied no text at all. + /// + /// The Agent reported UNAVAILABLE, or supplied no text at all for a non-string item + /// (an empty string-typed item is a Good "" — lmxopcua#824). + /// private const uint BadNoCommunication = 0x80310000u; /// The tag is authored but the Agent has not yet reported it. @@ -443,10 +446,8 @@ internal sealed class MTConnectObservationIndex var serverTimestamp = Now; var sourceTimestamp = DateTime.SpecifyKind(observation.TimestampUtc, DateTimeKind.Utc); - // No text at all is the degenerate spelling of "no value": MTConnect defines UNAVAILABLE as - // the only way to report absence, so an empty element is not an empty string value. - if (string.IsNullOrWhiteSpace(observation.Value) || - string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal)) + // UNAVAILABLE is MTConnect's one explicit "no value" sentinel — always BadNoCommunication. + if (string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal)) { return new DataValueSnapshot(null, BadNoCommunication, sourceTimestamp, serverTimestamp); } @@ -456,7 +457,7 @@ internal sealed class MTConnectObservationIndex // Publishing that as a Good string would be noise an operator would trust. Checked before // the tag lookup because it is a fact about the wire shape, true whether or not the data // item is authored — and after the sentinel, so an unavailable data set still reports the - // truthful no-comms. + // truthful no-comms, but BEFORE the empty-text test (#824). if (observation.IsStructured) { return new DataValueSnapshot(null, BadNotSupported, sourceTimestamp, serverTimestamp); @@ -464,6 +465,24 @@ internal sealed class MTConnectObservationIndex _tags.TryGetValue(observation.DataItemId, out var tag); + // A genuinely EMPTY element (no text, or whitespace the parser trimmed away) is NOT the + // sentinel. Owner decision on lmxopcua#824 (Java parity): on a string-typed item it is a + // Good empty string — an Agent can legitimately report a blank Program/Message/Block. + // An unauthored item coerces as String (below), so it is served the same way. + // On any other type (numeric, Boolean, DateTime, any array) "" cannot be a value, so it + // stays Bad — and stays BadNoCommunication rather than BadTypeMismatch on purpose: + // the Agent supplied no value at all (the same meaning as UNAVAILABLE, just spelled + // degenerately), whereas BadTypeMismatch tells the operator the Agent sent text of the + // wrong type and points them at the tag's authored data type, which is not the fault. + // Checked AFTER the structured test above, so element-children with no text stay + // BadNotSupported rather than becoming a Good "". + if (string.IsNullOrWhiteSpace(observation.Value)) + { + return tag is null or { IsArray: false, DriverDataType: DriverDataType.String or DriverDataType.Reference } + ? new DataValueSnapshot(string.Empty, Good, sourceTimestamp, serverTimestamp) + : new DataValueSnapshot(null, BadNoCommunication, sourceTimestamp, serverTimestamp); + } + // TIME_SERIES vectors arrive as one whitespace-separated string. Checked AFTER the // sentinel so an unavailable array tag still reports the truthful no-comms. if (tag is { IsArray: true }) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectObservationIndexTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectObservationIndexTests.cs index d8aab2f87..f064f6ff7 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectObservationIndexTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectObservationIndexTests.cs @@ -320,21 +320,107 @@ public sealed class MTConnectObservationIndexTests .ShouldBe(new DateTime(2026, 7, 24, 12, 0, 0, 500, DateTimeKind.Utc)); } - /// An observation whose text is empty is "the Agent supplied no value" — never a Good empty string. + /// + /// lmxopcua#824 (Java parity): an empty element on a string-typed item is a Good empty string, not an + /// absence. Whitespace-only text is the same (the parser trims it). + /// [Theory] - [InlineData("")] - [InlineData(" ")] - public void Empty_observation_value_maps_to_BadNoCommunication(string value) + [InlineData("", DriverDataType.String)] + [InlineData(" ", DriverDataType.String)] + [InlineData("", DriverDataType.Reference)] + public void Empty_observation_value_on_a_string_item_is_a_Good_empty_string(string value, DriverDataType type) { - var idx = new MTConnectObservationIndex([Tag("x", DriverDataType.String)]); + var idx = new MTConnectObservationIndex([Tag("x", type)]); idx.Apply(Synthetic("x", value)); var snap = idx.Get("x"); + snap.StatusCode.ShouldBe(Good); + snap.Value.ShouldBe(string.Empty); + } + + /// An unauthored item coerces as String, so its empty value is a Good empty string too (#824). + [Fact] + public void Empty_observation_value_on_an_unauthored_item_is_a_Good_empty_string() + { + var idx = new MTConnectObservationIndex(); + idx.Apply(Synthetic("x", "")); + + var snap = idx.Get("x"); + + snap.StatusCode.ShouldBe(Good); + snap.Value.ShouldBe(string.Empty); + } + + /// + /// #824: on a non-string item "" cannot be a value, so it stays BadNoCommunication (the Agent supplied + /// no value) — not BadTypeMismatch, which would blame the tag's authored type. + /// + [Theory] + [InlineData(DriverDataType.Float64)] + [InlineData(DriverDataType.Float32)] + [InlineData(DriverDataType.Int32)] + [InlineData(DriverDataType.Int64)] + [InlineData(DriverDataType.Boolean)] + [InlineData(DriverDataType.DateTime)] + public void Empty_observation_value_on_a_non_string_item_stays_BadNoCommunication(DriverDataType type) + { + var idx = new MTConnectObservationIndex([Tag("x", type)]); + idx.Apply(Synthetic("x", " ")); + + var snap = idx.Get("x"); + snap.StatusCode.ShouldBe(BadNoCommunication); snap.Value.ShouldBeNull(); } + /// + /// #824 control: the UNAVAILABLE sentinel is unaffected — still BadNoCommunication on a String item, where an + /// empty value is now Good. + /// + [Fact] + public void Unavailable_on_a_string_item_is_still_BadNoCommunication() + { + var idx = new MTConnectObservationIndex([Tag("x", DriverDataType.String)]); + idx.Apply(Synthetic("x", "UNAVAILABLE")); + + var snap = idx.Get("x"); + + snap.StatusCode.ShouldBe(BadNoCommunication); + snap.Value.ShouldBeNull(); + } + + /// #824 end to end through the parser: an empty, a self-closing and a blank element are all Good "". + [Theory] + [InlineData("""""")] + [InlineData("""""")] + [InlineData(""" """)] + public void A_parsed_empty_event_on_a_string_item_is_a_Good_empty_string(string element) + { + var idx = new MTConnectObservationIndex([Tag("p", DriverDataType.String)]); + idx.Apply(MTConnectStreamsParser.Parse($""" + +
+ {element} + + """)); + + var snap = idx.Get("p"); + + snap.StatusCode.ShouldBe(Good); + snap.Value.ShouldBe(string.Empty); + } + + /// A structured observation with no text stays BadNotSupported — it never becomes a Good "" (#824). + [Fact] + public void A_structured_observation_with_no_text_stays_BadNotSupported() + { + var idx = new MTConnectObservationIndex([Tag("x", DriverDataType.String)]); + idx.Apply(Synthetic("x", "", isStructured: true)); + + idx.Get("x").StatusCode.ShouldBe(BadNotSupported); + } + // -------------------------------------------------------------------------- unknown lookups /// A dataItemId that is neither authored nor ever observed is unknown to this driver. From e6ec5a09486e1c835ca2fa8f79d2aadebd288612 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 11:57:59 -0500 Subject: [PATCH 02/28] docs(opcuaclient): keep server host-name check opt-in; surface it in the Security panel (#837) Owner decision 2026-11-11: certificateValidation.validateServerHostName stays opt-in (unset = off). The AdminUI toggle moves from the "Subscriptions, limits & discovery" panel to the Security panel beside mode/policy, with help text on the NAT/IP caveat, IPv6 zone ids and the loose first-label match. Binding unchanged (tri-state, "Default (off)" leaves the key absent). OpcUaClient.md states the decision; drivers rule updated; source-scan test pins placement + caveats. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .claude/rules/drivers.md | 3 +- docs/drivers/OpcUaClient.md | 12 +++++-- .../Drivers/Forms/OpcUaClientDriverForm.razor | 33 ++++++++++++++----- ...aClientDriverPageFormSerializationTests.cs | 28 ++++++++++++++++ 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/.claude/rules/drivers.md b/.claude/rules/drivers.md index 8cfe9688a..8586e567e 100644 --- a/.claude/rules/drivers.md +++ b/.claude/rules/drivers.md @@ -293,7 +293,8 @@ host logger factory (wired in `DriverFactoryBootstrap`; the FOCAS and TwinCAT fa - **OpcUaClient server host-name check (#816):** the SDK checks the server certificate's host only with `checkDomain: true`; its 8-arg `DefaultSessionFactory.CreateAsync` hard-codes `false`, so driver, AdminUI browser and Client.Shared never checked before #816. Now `certificateValidation.validateServerHostName` (tri-state, unset = off = - unchanged; AdminUI keeps it unset) / CLI `--validate-host-name`. It checks the **configured** host (the rewritten + unchanged; AdminUI keeps it unset; **stays opt-in by owner decision #837** — the toggle lives in the form's Security +panel with the NAT/IPv6-zone/loose-match caveats) / CLI `--validate-host-name`. It checks the **configured** host (the rewritten `ConfiguredEndpoint.EndpointUrl`) and auto-accept accepts a mismatch. ⚠️ The SDK checks in TWO places — `DefaultSessionFactory.CreateChannelAsync` (GetEndpoints cert, BEFORE any security-mode test, so it fired on `None`) and `Session.OpenAsync` — so pass `checkDomain` only for a non-`None` endpoint (`OpcUaEndpointSelection.CheckDomainFor`). diff --git a/docs/drivers/OpcUaClient.md b/docs/drivers/OpcUaClient.md index 677493e6a..fd7014681 100644 --- a/docs/drivers/OpcUaClient.md +++ b/docs/drivers/OpcUaClient.md @@ -189,8 +189,16 @@ Consequences: - Test Connect (`OpcUaClientDriverProbe`) only calls GetEndpoints on an unsecured discovery channel — no session, so no certificate or host-name check. -**Policy: `certificateValidation.validateServerHostName`** (driver JSON; AdminUI driver form "Server host-name check"; -the browser reads the same JSON): +**Decision (#837, owner, 2026-11-11): the check stays OPT-IN — unset means off, and it will not become a default.** +Turning it on by default would refuse every NAT / port-published / connect-by-IP endpoint whose certificate does not +list that exact host, and every link-local IPv6 endpoint, on upgrade; and because the match is loose (first label) it +would not deliver the identity guarantee a default implies. Operators enable it per driver where trust is CA-issued +and certificate names are fully-qualified and site-unique. The AdminUI shows the toggle in the driver form's +**Security** panel (beside mode/policy) as "Server certificate host-name check", with help text covering the NAT/IP +caveat, IPv6 zone ids and the loose first-label match. + +**Policy: `certificateValidation.validateServerHostName`** (driver JSON; AdminUI driver form, Security panel, +"Server certificate host-name check"; the browser reads the same JSON): | Value | Meaning | |---|---| diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/OpcUaClientDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/OpcUaClientDriverForm.razor index f52566321..1cbb7f2bd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/OpcUaClientDriverForm.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/OpcUaClientDriverForm.razor @@ -111,6 +111,30 @@ } + @* #816 / #837: opt-in by owner decision (unset = off). Kept in the Security panel so it is + found next to the mode/policy it depends on; "Default (off)" leaves the key absent. *@ +
+ + + + + + +
+
+
+ Opt-in (off unless set). When validating, the server certificate must name the + endpoint host you configured; a miss refuses the connection + (BadCertificateHostNameInvalid). Applies to Sign / SignAndEncrypt only, never to + None, and auto-accept accepts a mismatch. Caveats: +
    +
  • NAT / port publish / connecting by IP: fails unless the certificate lists that exact host or IP — leave off.
    • +
  • IPv6 zone ids (fe80::1%eth0, link-local) never match — leave off.
    • +
  • Loose match: the first DNS label alone counts (plc1.site-b.corp passes a certificate for PLC1), so use fully-qualified, site-unique certificate names.
    • +
+ Most useful with CA-issued trust; adds little over a pinned self-signed certificate. +
+
@@ -195,15 +219,6 @@
Revoked certificates are always rejected.
-
- - - - - - -
Sign / SignAndEncrypt only (never on None): the server certificate must name the endpoint host you configured. Off for link-local IPv6.
-
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/OpcUaClientDriverPageFormSerializationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/OpcUaClientDriverPageFormSerializationTests.cs index af598b20e..8774f94ba 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/OpcUaClientDriverPageFormSerializationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/OpcUaClientDriverPageFormSerializationTests.cs @@ -208,6 +208,34 @@ public sealed class OpcUaClientDriverPageFormSerializationTests reparsed.CertificateValidation.EffectiveValidateServerHostName.ShouldBe(value ?? false); } + /// + /// #837: the host-name check stays opt-in (unset = off) and sits in the Security panel with help text naming + /// the NAT/IP, IPv6-zone and loose-first-label caveats. Source scan (no bUnit in this repo). + /// + [Fact] + public void The_host_name_check_is_an_opt_in_in_the_security_panel_with_its_caveats() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.OtOpcUa.slnx"))) + dir = dir.Parent; + dir.ShouldNotBeNull("could not locate the repository root"); + var razor = File.ReadAllText(Path.Combine(dir!.FullName, "src", "Server", "ZB.MOM.WW.OtOpcUa.AdminUI", + "Components", "Shared", "Drivers", "Forms", "OpcUaClientDriverForm.razor")); + + razor.ShouldContain(""); + + var security = razor.IndexOf("
Security
", StringComparison.Ordinal); + var next = razor.IndexOf("
", security + 1, StringComparison.Ordinal); + security.ShouldBeGreaterThan(-1); + var panel = razor[security..next]; + panel.ShouldContain("_form.OpcUa.ValidateServerHostName"); + panel.ShouldContain("Opt-in"); + panel.ShouldContain("NAT"); + panel.ShouldContain("IPv6 zone ids"); + panel.ShouldContain("Loose match"); + razor.Split("_form.OpcUa.ValidateServerHostName").Length.ShouldBe(2, "the toggle must be rendered once"); + } + /// /// #648: stored DriverConfig JSON from before targetNamespaceKind/unsMappingTable were removed /// must still load into the form (the form's options use UnmappedMemberHandling.Skip) and the legacy From 43bdeff2b647e3afe964f5c9fe78404787b7bf69 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 11:58:35 -0500 Subject: [PATCH 03/28] fix(adminui): Faulted-devices tile counts distinct Faulted devices across the cluster pair (#786) Owner decision 2026-11-11: /fleet/diagnostics "Faulted devices" counts DISTINCT devices across the nodes of the selected node's cluster, Faulted state only. - DriverHostActor: FaultedDevices = HostState.Faulted only (Stopped no longer counted); new StoppedDevices + FaultedHostNames. Counting extracted to the testable DiagnosticsFromHealth. - DriverInstanceDiagnostics: StoppedDevices (default 0) and FaultedHostNames (default null) are additive with defaults, so a pre-#786 sender's snapshot still deserializes over Akka's default JSON serializer (pinned by a test). - FleetDiagnosticsAggregation (pure): identity key = (driver-instance string id, HostName), case-insensitive; a single-host driver / nameless pre-#786 count is combined by MAX across nodes, never summed. Cluster peers come from enabled ClusterNode rows of the selected node's ClusterId. - FleetDiagnostics.razor: tile fetches the cluster peers and shows the distinct count + "n of m nodes answered"; per-driver table gains a Stopped devices column. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../Interfaces/NodeDiagnosticsSnapshot.cs | 24 ++- .../Clients/FleetDiagnosticsAggregation.cs | 117 +++++++++++++ .../Components/Pages/FleetDiagnostics.razor | 65 ++++++- .../Drivers/DriverHostActor.cs | 33 +++- .../FleetDiagnosticsAggregationTests.cs | 165 ++++++++++++++++++ .../DriverHostActorDiagnosticsTests.cs | 43 +++++ 6 files changed, 437 insertions(+), 10 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Clients/FleetDiagnosticsAggregation.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Interfaces/NodeDiagnosticsSnapshot.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Interfaces/NodeDiagnosticsSnapshot.cs index a2fafebd5..f8db2d319 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Interfaces/NodeDiagnosticsSnapshot.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Interfaces/NodeDiagnosticsSnapshot.cs @@ -2,13 +2,35 @@ using ZB.MOM.WW.OtOpcUa.Commons.Types; namespace ZB.MOM.WW.OtOpcUa.Commons.Interfaces; +/// One driver child's row in a . +/// The driver instance's ConfigDb row id. +/// The string driver-instance id (the identity every other surface keys on). +/// The child's last-published health state, or "Connecting" / "Stubbed". +/// Hosts in Running (or 1 for a Healthy/Degraded single-host driver). +/// +/// Hosts in Faulted only (or 1 for a Faulted single-host driver). Since #786 a Stopped host is +/// NOT counted here — it is . A sender built before #786 still counts +/// Stopped hosts in this figure. +/// +/// When the child last published a changed health snapshot (its spawn time until the first). +/// +/// Hosts in Stopped (#786). Additive with a default so a snapshot from an older sender — which has no +/// such member on the wire — still deserializes, reading 0. +/// +/// +/// The HostNames of the hosts counted in (#786), so the AdminUI can +/// count a device faulted on BOTH nodes of a redundant pair once. Empty for a single-host driver (the driver +/// itself is the device); from a sender built before #786. +/// public sealed record DriverInstanceDiagnostics( Guid DriverInstanceId, string Name, string State, int ConnectedDevices, int FaultedDevices, - DateTime LastChangeUtc); + DateTime LastChangeUtc, + int StoppedDevices = 0, + IReadOnlyList? FaultedHostNames = null); /// /// Per-node diagnostics returned by IFleetDiagnosticsClient. Populated by the node's diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Clients/FleetDiagnosticsAggregation.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Clients/FleetDiagnosticsAggregation.cs new file mode 100644 index 000000000..770357abc --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Clients/FleetDiagnosticsAggregation.cs @@ -0,0 +1,117 @@ +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Clients; + +/// +/// Cluster-wide counting for the /fleet/diagnostics "Faulted devices" tile (#786). Pure, so it is +/// unit-testable without bUnit. +/// +/// +/// +/// Owner decision 2026-11-11: the tile counts DISTINCT devices across the nodes of one application +/// cluster (a redundant pair), and only devices in Faulted — Stopped is a separate per-node +/// column. Both nodes of a pair run the same driver instances against the same devices, so summing the two +/// nodes' per-driver counts reported every faulted device twice. +/// +/// +/// The device identity key is (driver-instance id, host name), both compared case-insensitively: +/// +/// +/// The driver-instance id is — the string id from the +/// deployed artifact, identical on both nodes of a pair (the row Guid is identical too, but can be +/// on a node with no ConfigDb row behind its spec, so the string id is the +/// sturdier key and the one every other surface uses). +/// +/// +/// The host name is the driver's own HostConnectivityStatus.HostName (e.g. host:port +/// for Modbus/S7, a Platform/Engine name for Galaxy) — driver-defined from the same authored config, +/// so the same device yields the same name on both nodes. It is scoped by the driver id because two +/// different drivers can legitimately name the same endpoint and are counted as two devices. +/// +/// +/// A single-host driver (no IHostConnectivityProbe) reports no host names: the driver instance itself +/// is the device. Its faulted figure (0 or 1) — and any count a pre-#786 sender reports without names — +/// is "unnamed", and unnamed counts for one driver are combined by MAX across nodes, never summed: the +/// same unnamed device on both nodes counts once, which is the most the snapshot can prove distinct. +/// +/// +/// A node that did not answer (the client degrades to an empty snapshot) contributes nothing, so the figure +/// is a lower bound over the nodes that did — the page shows how many answered. +/// +/// +public static class FleetDiagnosticsAggregation +{ + /// Counts the distinct faulted devices across a set of node snapshots (normally one cluster's nodes). + /// One snapshot per node; duplicates of a node are harmless (the count is a set union). + /// The number of distinct (driver id, host name) devices faulted on at least one node. + public static int DistinctFaultedDevices(IEnumerable snapshots) + { + ArgumentNullException.ThrowIfNull(snapshots); + + var named = new HashSet<(string Driver, string Host)>(DeviceKeyComparer.Instance); + var unnamedMax = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var snapshot in snapshots) + { + foreach (var driver in snapshot.Drivers) + { + var names = driver.FaultedHostNames ?? Array.Empty(); + var namedHere = 0; + foreach (var host in names.Where(n => !string.IsNullOrWhiteSpace(n)).Distinct(StringComparer.OrdinalIgnoreCase)) + { + named.Add((driver.Name, host)); + namedHere++; + } + + var unnamed = Math.Max(0, driver.FaultedDevices - namedHere); + if (unnamed > 0) + unnamedMax[driver.Name] = Math.Max(unnamedMax.GetValueOrDefault(driver.Name), unnamed); + } + } + + return named.Count + unnamedMax.Values.Sum(); + } + + /// + /// The node ids (host:port) of every enabled node in the same application cluster as + /// , the node itself included. A node with no ClusterNode row (or an empty + /// ConfigDb) yields just itself, so the tile degrades to a single-node count rather than failing. + /// + /// The config DB context. + /// The selected node's id, as /fleet shows it. + /// Cancellation token. + /// The cluster id (null when unknown) and its node ids, the selected node first. + public static async Task<(string? ClusterId, IReadOnlyList NodeIds)> LoadClusterNodeIdsAsync( + OtOpcUaConfigDbContext db, string nodeId, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + var self = await db.ClusterNodes.AsNoTracking() + .Where(n => n.NodeId == nodeId) + .Select(n => n.ClusterId) + .FirstOrDefaultAsync(ct); + if (self is null) return (null, new[] { nodeId }); + + var peers = await db.ClusterNodes.AsNoTracking() + .Where(n => n.ClusterId == self && n.Enabled && n.NodeId != nodeId) + .Select(n => n.NodeId) + .ToListAsync(ct); + peers.Sort(StringComparer.OrdinalIgnoreCase); + return (self, new[] { nodeId }.Concat(peers).ToList()); + } + + private sealed class DeviceKeyComparer : IEqualityComparer<(string Driver, string Host)> + { + public static readonly DeviceKeyComparer Instance = new(); + + public bool Equals((string Driver, string Host) x, (string Driver, string Host) y) => + StringComparer.OrdinalIgnoreCase.Equals(x.Driver, y.Driver) + && StringComparer.OrdinalIgnoreCase.Equals(x.Host, y.Host); + + public int GetHashCode((string Driver, string Host) key) => + HashCode.Combine( + StringComparer.OrdinalIgnoreCase.GetHashCode(key.Driver), + StringComparer.OrdinalIgnoreCase.GetHashCode(key.Host)); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/FleetDiagnostics.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/FleetDiagnostics.razor index e55791b73..b8256cb3e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/FleetDiagnostics.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/FleetDiagnostics.razor @@ -86,9 +86,17 @@ else
Drivers
@_snapshot.Drivers.Count
-
-
Faulted devices
-
@_snapshot.Drivers.Sum(d => d.FaultedDevices)
+ @* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on + both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below. + See FleetDiagnosticsAggregation for the identity key. *@ +
+
Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")
+
@_clusterFaulted
+ @if (_clusterNodeCount > 1) + { +
@_clusterAnswered of @_clusterNodeCount nodes answered
+ }
@@ -124,6 +132,7 @@ else State Connected devices Faulted devices + Stopped devices Last health publish @@ -144,6 +153,7 @@ else @d.State @d.ConnectedDevices @d.FaultedDevices + @d.StoppedDevices @d.LastChangeUtc.ToString("u") } @@ -165,6 +175,12 @@ else private NodeDiagnosticsSnapshot? _snapshot; private IReadOnlyDictionary _displayNames = new Dictionary(); private bool _loading; + // #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the + // distinct faulted-device count across them. + private string? _clusterId; + private int _clusterNodeCount; + private int _clusterAnswered; + private int _clusterFaulted; private DateTime? _fetchedUtc; // Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned. @@ -222,6 +238,7 @@ else // The client is fail-safe by contract: a node that does not answer yields an empty snapshot. _snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token); _fetchedUtc = DateTime.UtcNow; + await LoadClusterFaultedAsync(_snapshot); _displayNames = await LoadDisplayNamesAsync(_snapshot); } catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException) @@ -235,6 +252,48 @@ else } } + // #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them. + // The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone. + private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected) + { + string? clusterId = null; + IReadOnlyList nodeIds = new[] { selected.NodeId.Value }; + try + { + await using var db = await DbFactory.CreateDbContextAsync(_cts.Token); + (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token); + } + catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException) + { + // ConfigDb hiccup — count this node only. + } + + var snapshots = new List { selected }; + var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList(); + var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync)); + snapshots.AddRange(peerSnapshots.OfType()); + + _clusterId = clusterId; + _clusterNodeCount = nodeIds.Count; + // The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty + // answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way. + _clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 }); + _clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots); + } + + private async Task FetchPeerAsync(string nodeId) + { + try + { + return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token); + } + catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException) + { + // A malformed peer row (NodeId.Parse) — the peer just does not contribute. + return null; + } + } + // The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed). private string DisplayNameOf(DriverInstanceDiagnostics d) => _displayNames.TryGetValue(d.Name, out var name) ? name : d.Name; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 3d58971c9..254a7ee21 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -2755,9 +2755,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// One child's diagnostics row. Name carries the string driver-instance id — the identity every /// other surface (/hosts, telemetry, the artifact) keys on — and DriverInstanceId the DB row Guid; /// the wire record has no slot for the authored display name. Device counts come from the driver's host - /// statuses when it is an IHostConnectivityProbe (Running = connected; Stopped/Faulted = faulted; - /// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or - /// Degraded and faulted while it reports Faulted. A child that has not published yet reports + /// statuses when it is an IHostConnectivityProbe (Running = connected; Faulted = faulted; + /// Stopped = stopped, counted separately since #786; Unknown = none) and the faulted hosts' names ride + /// along so the AdminUI can de-duplicate a device across a redundant pair; a single-host driver counts as + /// one device, connected while it reports Healthy or Degraded and faulted while it reports Faulted. A child that has not published yet reports /// "Connecting" (or "Stubbed") with its spawn time as the last change. /// private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health) @@ -2768,20 +2769,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS if (health is null) return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc); - int connected, faulted; + return DiagnosticsFromHealth(entry.Spec.DriverInstanceRowId, name, health); + } + + /// The device counts of a child that has published health (#786: Faulted and Stopped split). + /// The driver instance's ConfigDb row id. + /// The string driver-instance id. + /// The child's last-published health. + /// The diagnostics row. + internal static DriverInstanceDiagnostics DiagnosticsFromHealth(Guid rowId, string name, ChildHealthRecord health) + { + int connected, faulted, stopped; + IReadOnlyList faultedHostNames; if (health.HostStatuses is { Count: > 0 } hosts) { connected = hosts.Count(h => h.State == HostState.Running); - faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted); + // #786: Faulted only. A Stopped host (e.g. a Galaxy engine an operator took off scan) is a + // deliberate, separately-reported state, not a fault. + var faultedHosts = hosts.Where(h => h.State == HostState.Faulted).ToArray(); + faulted = faultedHosts.Length; + stopped = hosts.Count(h => h.State == HostState.Stopped); + faultedHostNames = faultedHosts.Select(h => h.HostName).ToArray(); } else { connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0; faulted = health.State == DriverState.Faulted ? 1 : 0; + stopped = 0; + // A single-host driver is its own device: no host name; the AdminUI keys it on the instance id. + faultedHostNames = Array.Empty(); } return new DriverInstanceDiagnostics( - entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc); + rowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc, + StoppedDevices: stopped, FaultedHostNames: faultedHostNames); } private void HandleDispatchFromSteady(DispatchDeployment msg) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs new file mode 100644 index 000000000..0eb62f8e6 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs @@ -0,0 +1,165 @@ +using System.Text; +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.AdminUI.Clients; +using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests; + +/// +/// #786: the /fleet/diagnostics "Faulted devices" tile counts DISTINCT faulted devices across a +/// cluster's nodes (a device faulted on both nodes of a redundant pair counts once), Faulted only. +/// +[Trait("Category", "Unit")] +public sealed class FleetDiagnosticsAggregationTests +{ + private static readonly DateTime Now = DateTime.UtcNow; + + private static DriverInstanceDiagnostics Driver( + string name, int faulted, IReadOnlyList? faultedHosts, int stopped = 0) => + new(Guid.NewGuid(), name, "Degraded", 0, faulted, Now, stopped, faultedHosts); + + private static NodeDiagnosticsSnapshot Node(string nodeId, params DriverInstanceDiagnostics[] drivers) => + new(NodeId.Parse(nodeId), null, drivers, Now); + + [Fact] + public void The_same_device_faulted_on_both_nodes_of_a_pair_counts_once() + { + var a = Node("node-a:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" })); + var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "PLC-1:502" })); // host names compare case-insensitively + + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2); + } + + [Fact] + public void Different_devices_faulted_on_each_node_are_unioned() + { + var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" })); + var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "plc-2:502" })); + + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2); + } + + [Fact] + public void The_same_host_name_under_two_drivers_is_two_devices() + { + var a = Node("node-a:4053", + Driver("drv-modbus", 1, new[] { "10.0.0.5" }), + Driver("drv-s7", 1, new[] { "10.0.0.5" })); + + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(2); + } + + [Fact] + public void A_faulted_single_host_driver_on_both_nodes_counts_once() + { + // No IHostConnectivityProbe: the driver itself is the device and carries no host name. + var a = Node("node-a:4053", Driver("drv-galaxy", 1, Array.Empty())); + var b = Node("node-b:4053", Driver("drv-galaxy", 1, Array.Empty())); + + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(1); + } + + [Fact] + public void Stopped_devices_are_not_counted() + { + var a = Node("node-a:4053", Driver("drv-galaxy", 0, Array.Empty(), stopped: 3)); + + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(0); + } + + [Fact] + public void A_pre_786_sender_without_host_names_is_combined_by_max_not_sum() + { + var oldSender = Node("node-a:4053", Driver("drv-modbus", 3, faultedHosts: null)); + var newSender = Node("node-b:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" })); + + // Unnamed 3 (max over nodes) + named 2: the old sender's devices cannot be matched by name, so the + // figure is the best the wire can prove, never a doubled sum of the same unnamed devices. + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, oldSender }).ShouldBe(3); + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, newSender }).ShouldBe(5); + } + + [Fact] + public void An_unreachable_node_empty_snapshot_contributes_nothing() + { + var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" })); + var unreachable = Node("node-b:4053"); + + FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, unreachable }).ShouldBe(1); + FleetDiagnosticsAggregation.DistinctFaultedDevices(Array.Empty()).ShouldBe(0); + } + + [Fact] + public async Task Cluster_node_ids_are_the_enabled_rows_of_the_selected_nodes_cluster_self_first() + { + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options; + await using (var seed = new OtOpcUaConfigDbContext(options)) + { + seed.ClusterNodes.AddRange( + Row("site-a-2:4053", "SITE-A"), + Row("site-a-1:4053", "SITE-A"), + Row("site-a-3:4053", "SITE-A", enabled: false), + Row("site-b-1:4053", "SITE-B")); + await seed.SaveChangesAsync(TestContext.Current.CancellationToken); + } + + await using var db = new OtOpcUaConfigDbContext(options); + var (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync( + db, "site-a-2:4053", TestContext.Current.CancellationToken); + clusterId.ShouldBe("SITE-A"); + nodeIds.ShouldBe(new[] { "site-a-2:4053", "site-a-1:4053" }); + + var (unknownCluster, alone) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync( + db, "stray:4053", TestContext.Current.CancellationToken); + unknownCluster.ShouldBeNull(); + alone.ShouldBe(new[] { "stray:4053" }); + } + + [Fact] + public async Task A_snapshot_from_a_pre_786_sender_still_deserializes_over_the_akka_wire() + { + // The snapshot crosses Akka remoting with the default (Newtonsoft JSON) serializer. Strip the #786 + // members from a serialized snapshot to reproduce an older sender's bytes. + using var system = ActorSystem.Create("diag-compat"); + try + { + var snapshot = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }, stopped: 2)); + var serializer = system.Serialization.FindSerializerFor(snapshot); + var json = Encoding.UTF8.GetString(serializer.ToBinary(snapshot)); + json.ShouldContain("StoppedDevices"); + + var tree = JObject.Parse(json); + foreach (var prop in tree.Descendants().OfType() + .Where(p => p.Name is "StoppedDevices" or "FaultedHostNames").ToList()) + prop.Remove(); + var oldJson = tree.ToString(Formatting.None); + oldJson.ShouldNotContain("StoppedDevices"); + oldJson.ShouldNotContain("FaultedHostNames"); + + var back = (NodeDiagnosticsSnapshot)serializer.FromBinary(Encoding.UTF8.GetBytes(oldJson), typeof(NodeDiagnosticsSnapshot)); + var driver = back.Drivers.ShouldHaveSingleItem(); + driver.FaultedDevices.ShouldBe(1); + driver.StoppedDevices.ShouldBe(0); + driver.FaultedHostNames.ShouldBeNull(); + } + finally + { + await system.Terminate(); + } + } + + private static ClusterNode Row(string nodeId, string clusterId, bool enabled = true) => new() + { + NodeId = nodeId, ClusterId = clusterId, Host = nodeId.Split(':')[0], + ApplicationUri = "urn:" + nodeId, CreatedBy = "test", Enabled = enabled, + }; +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs index 86ca424b0..f1b6294ea 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs @@ -4,6 +4,8 @@ using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet; using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; @@ -43,6 +45,7 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase driver.State.ShouldBe("Healthy"); driver.ConnectedDevices.ShouldBe(1); driver.FaultedDevices.ShouldBe(0); + driver.StoppedDevices.ShouldBe(0); driver.LastChangeUtc.ShouldBeGreaterThan(before); driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow); } @@ -66,6 +69,46 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase driver.FaultedDevices.ShouldBe(0); } + [Fact] + public void Faulted_counts_only_Faulted_hosts_and_Stopped_is_reported_separately() + { + // #786: a Stopped host used to be counted as faulted. + var now = DateTime.UtcNow; + var health = new ChildHealthRecord( + DriverState.Degraded, now, null, + new[] + { + new HostConnectivityStatus("plc-a:502", HostState.Running, now), + new HostConnectivityStatus("plc-b:502", HostState.Faulted, now), + new HostConnectivityStatus("plc-c:502", HostState.Stopped, now), + new HostConnectivityStatus("plc-d:502", HostState.Stopped, now), + new HostConnectivityStatus("plc-e:502", HostState.Unknown, now), + }, + now); + + var row = DriverHostActor.DiagnosticsFromHealth(Guid.NewGuid(), DriverId, health); + + row.ConnectedDevices.ShouldBe(1); + row.FaultedDevices.ShouldBe(1); + row.StoppedDevices.ShouldBe(2); + row.FaultedHostNames.ShouldBe(new[] { "plc-b:502" }); + } + + [Theory] + [InlineData(DriverState.Faulted, 0, 1)] + [InlineData(DriverState.Healthy, 1, 0)] + [InlineData(DriverState.Degraded, 1, 0)] + public void A_single_host_driver_is_one_device_with_no_host_name(DriverState state, int connected, int faulted) + { + var row = DriverHostActor.DiagnosticsFromHealth( + Guid.NewGuid(), DriverId, new ChildHealthRecord(state, null, null, null, DateTime.UtcNow)); + + row.ConnectedDevices.ShouldBe(connected); + row.FaultedDevices.ShouldBe(faulted); + row.StoppedDevices.ShouldBe(0); + row.FaultedHostNames.ShouldNotBeNull().ShouldBeEmpty(); + } + private async Task AwaitDiagnosticsAsync(IActorRef host, Func ready) { NodeDiagnosticsSnapshot? snapshot = null; From ed05d50795603bbc8e46ebfaac3cf9836b15e85e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:05:34 -0500 Subject: [PATCH 04/28] fix(aggregates): live PercentGood/DurationGood count Good time only (#809) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frozen 1.5.378.182 StatusAggregateCalculator counts Uncertain regions as Good under TreatUncertainAsBad=false, so live AggregateFilter PercentGood / DurationGood disagreed with HistoryRead (Part 13 §5.4.3.31/§5.4.3.33, Mantis 0006166) whenever Uncertain data was present. Owner-approved 2026-11-11. - New app-owned GoodOnlyStatusAggregateCalculator subclasses the baseline and replaces only the PercentGood/DurationGood computation (Good regions only); CompatibilityAggregateFactory routes exactly those two IDs to it. The copied SDK files are untouched; the other 35 aggregates are byte-identical. - Golden test: the 64 changed vectors (38 PercentGood + 26 DurationGood) are exempted exactly — those whose baseline output differs from the same case's baseline TreatUncertainAsBad=true twin — and pinned to that twin (baseline evidence, not regenerated from the implementation); plus a full Good-only pin over all 352 PercentGood/DurationGood vectors and a hand-computed case. - Live-vs-history parity over Good/Uncertain/Bad mixes (no masks), with a discrimination check against the pre-#809 baseline; the booted-runtime facts test now expects 8000 ms / 40 %. - SdkAggregateOracle: Run/RunLive = the registered factory; RunSdkAsIs = the untouched baseline; its Part13GoodCalculator duplicate is removed. - README (recorded decision), ServerAggregateConfiguration remarks, docs/Historian.md and .claude/rules/historian.md updated. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .claude/rules/historian.md | 2 +- docs/Historian.md | 14 +- .../CompatibilityAggregateFactory.cs | 10 +- .../GoodOnlyStatusAggregateCalculator.cs | 83 ++++++++ .../Compatibility/Aggregates/README.md | 11 + .../ServerAggregateConfiguration.cs | 14 +- .../QualityMixConformance.cs | 5 +- .../SdkAggregateOracle.cs | 93 ++++----- .../SdkAggregateParityTests.cs | 110 +++++++++- .../LiveAggregateLateStartFactsTests.cs | 14 +- .../Sdk2AggregateCompatibilityTests.cs | 191 +++++++++++++----- 11 files changed, 421 insertions(+), 126 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/GoodOnlyStatusAggregateCalculator.cs diff --git a/.claude/rules/historian.md b/.claude/rules/historian.md index 2d37cf743..7c8bf0050 100644 --- a/.claude/rules/historian.md +++ b/.claude/rules/historian.md @@ -32,7 +32,7 @@ historian tagname, which is the tag's driver `FullName`. ### Pluggable backends, SQL Server and HistoryUpdate (2026-10) - **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`. -- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; the live AggregateFilter still counts Uncertain as Good, #809) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. +- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; since #809 the live AggregateFilter agrees — `GoodOnlyStatusAggregateCalculator` overrides only those two of the frozen calculators) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates. - ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads". diff --git a/docs/Historian.md b/docs/Historian.md index 6d7fa9963..e160a56b1 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -1827,10 +1827,13 @@ is `Good` (plus Calculated / Partial). Value = `100 × region time ÷ interval d **`DurationGood` / `DurationBad`** (#820) are the same Good / Bad region times as `PercentGood` / `PercentBad` — every rule above (regions, simple bounds, time before the first sample = Bad, end of data, `BadNoData`, status, Partial) applies unchanged — reported as an OPC UA `Duration`, i.e. a `double` in **milliseconds** (the SDK's unit): `DurationGood = PercentGood × interval ms ÷ 100`. Uncertain -time counts toward neither (Mantis 0006166). Before #820 both were `BadAggregateNotSupported` on the SQL backends. ⚠️ A **live** -`AggregateFilter` subscription computes DurationGood/PercentGood with the frozen 1.5.378.182 calculators, which count Uncertain time as -Good — they disagree with HistoryRead whenever Uncertain data is present (#809, owner decision pending). The Gateway backend still -refuses both (`BadAggregateNotSupported`, not advertised in i=11201). +time counts toward neither (Mantis 0006166). Before #820 both were `BadAggregateNotSupported` on the SQL backends. ⚠️ **#809 +(client-visible, owner-approved 2026-11-11):** a **live** `AggregateFilter` subscription now computes DurationGood/PercentGood the same +way — Good time only. They used to come from the frozen 1.5.378.182 calculators, which count Uncertain time as Good; the server now +serves those two through the application's `GoodOnlyStatusAggregateCalculator` (every other live aggregate is still the frozen +baseline). A live PercentGood/DurationGood over Uncertain data therefore drops by the Uncertain share. Pinned live-vs-history by +`SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples`. The Gateway +backend still refuses both (`BadAggregateNotSupported`, not advertised in i=11201). All carry the Calculated bit. A non-Bad result also carries the **Partial** bit (`0x4`, `AggregateBits.Partial`) — never on a Bad result — for: the **truncated interval** (shorter than the processing interval — the latest one forward, the earliest one reverse; @@ -1911,7 +1914,8 @@ registers for **live** `AggregateFilter` items and from SDK 2.0.1's own stock ca `SdkAggregateParityTests.An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data` (ours / frozen / stock 2.0.1, with a Good-first control) and `OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests` (through the booted server's runtime `AggregateManager`), and on both SQL backends by `QualityMixConformance.Time_weighted_values_over_partial_data_cover_only_the_data` -(forward) and `…_in_a_reverse_read` (#821). Whether live subscriptions should follow is the open #809/#821 owner decision. The same +(forward) and `…_in_a_reverse_read` (#821). Whether live subscriptions should follow is the open #821 owner decision (#809, the +PercentGood/DurationGood half, is decided: live follows Part 13). The same suite feeds PercentGood/PercentBad, DurationGood/DurationBad (and Count, Average, TimeAverage(2), Total(2), Start/End/Delta, Interpolative) Good/Uncertain/Bad mixes, all-Uncertain intervals (0 % Good **and** 0 % Bad, Mantis 0006166) and partial-data intervals, hand-computed and against the #796 oracle — the baseline calculator's Uncertain-as-Good PercentGood fails it. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs index c4d1b6f9a..3be72a299 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs @@ -6,10 +6,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates; /// Adapts the versioned baseline algorithms to the SDK 2 calculator protocol without a second SDK runtime. internal static class CompatibilityAggregateFactory { + /// + /// PercentGood / DurationGood are served by the application-owned + /// (#809: Uncertain time is not Good, as in Part 13 and HistoryRead); every other id is the unchanged baseline. + /// internal static IAggregateCalculator Create(NodeId id, DateTimeUtc start, DateTimeUtc end, double interval, bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry) => - new Adapter(Aggregators.CreateStandardCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) - ?? throw new ArgumentOutOfRangeException(nameof(id), "Unsupported baseline aggregate.")); + new Adapter(GoodOnlyStatusAggregateCalculator.Overrides(id) + ? new GoodOnlyStatusAggregateCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) + : Aggregators.CreateStandardCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) + ?? throw new ArgumentOutOfRangeException(nameof(id), "Unsupported baseline aggregate.")); /// Replaces exactly the existing standard factories; no aggregate IDs are added or removed. internal static async ValueTask RegisterAsync(AggregateManager manager, CancellationToken ct) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/GoodOnlyStatusAggregateCalculator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/GoodOnlyStatusAggregateCalculator.cs new file mode 100644 index 000000000..f8532f7fd --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/GoodOnlyStatusAggregateCalculator.cs @@ -0,0 +1,83 @@ +using Opc.Ua; +using DataValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.LegacyDataValue; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates; + +/// +/// Application-owned override (#809) of the baseline for +/// PercentGood and DurationGood only: a region counts toward the Good time only when its status is +/// Good. Uncertain time is in neither Good nor (unless TreatUncertainAsBad) Bad, per Part 13 +/// §5.4.3.31/§5.4.3.33 (Mantis 0006166) — the rule the HistoryRead evaluators (TimeWeightedAggregator, +/// SqlServer + QuestDB) already follow. +/// +/// +/// The baseline 1.5.378.182 ComputeDurationGoodBad counts Uncertain regions as Good when +/// TreatUncertainAsBad = false. This class leaves the copied file untouched and replaces only the Good +/// branch: the slice values, simple bounds, stepped regions, empty-slice BadNoData, timestamp, value type +/// and Calculated status bit are the baseline's. DurationBad / PercentBad / WorstQuality* +/// fall through to the baseline unchanged. +/// +internal sealed class GoodOnlyStatusAggregateCalculator : StatusAggregateCalculator +{ + /// True for the two aggregate ids this class overrides. + internal static bool Overrides(NodeId aggregateId) => + aggregateId == ObjectIds.AggregateFunction_PercentGood || aggregateId == ObjectIds.AggregateFunction_DurationGood; + + /// Initializes the calculator; same contract as the baseline constructor. + public GoodOnlyStatusAggregateCalculator( + NodeId aggregateId, + DateTime startTime, + DateTime endTime, + double processingInterval, + bool stepped, + AggregateConfiguration configuration, + ITelemetryContext? telemetry) + : base(aggregateId, startTime, endTime, processingInterval, stepped, configuration, telemetry) + { + } + + /// + protected override DataValue ComputeValue(TimeSlice slice) + { + if (AggregateId == ObjectIds.AggregateFunction_DurationGood) return ComputeGoodOnly(slice, usePercent: false); + if (AggregateId == ObjectIds.AggregateFunction_PercentGood) return ComputeGoodOnly(slice, usePercent: true); + return base.ComputeValue(slice); + } + + /// The baseline ComputeDurationGoodBad(slice, isBad: false, usePercent) with Good-only counting. + private DataValue ComputeGoodOnly(TimeSlice slice, bool usePercent) + { + List? values = GetValuesWithSimpleBounds(slice); + if (values == null || values.Count == 0) + { + return GetNoDataValue(slice); + } + + List regions = GetRegionsInValueSet(values, false, true); + + double duration = 0; + double total = 0; + foreach (var region in regions) + { + total += region.Duration; + if (StatusCode.IsGood(region.StatusCode)) + { + duration += region.Duration; + } + } + + if (usePercent) + { + duration = duration / total * 100; + } + + var value = new DataValue + { + WrappedValue = SdkValueBoundary.FromClr(duration), + SourceTimestamp = GetTimestamp(slice), + ServerTimestamp = GetTimestamp(slice) + }; + value.StatusCode = value.StatusCode.WithAggregateBits(AggregateBits.Calculated); + return value; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md index 3f1ddc0b3..1ba0f0c95 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md @@ -17,3 +17,14 @@ The isolated `scripts/validation/opcua-aggregate-baseline` process pins only SDK Run the generator with a **new** output path, then compare it with `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Fixtures/opcua-1.5.378.182-aggregates.json`. It refuses to overwrite evidence. Never regenerate expectations from the compatibility implementation. Existing historian parity tests retain their application-specific documented divergences from the baseline SDK, while new compatibility tests check the complete old-calculator output. This preserves older behavior deliberately; it does not claim that all older SDK algorithms implement the newest OPC UA interpretation. A future semantic migration needs its own consumer-impact decision and versioned tests. + +## Recorded semantic decisions + +### #809 — PercentGood / DurationGood count Good time only (owner-approved 2026-11-11) + +The baseline `StatusAggregateCalculator.ComputeDurationGoodBad` counts Uncertain regions as **Good** when `TreatUncertainAsBad = false` (1.5.378.106 and SDK 2.0.1 count Good only). Part 13 §5.4.3.31/§5.4.3.33 (Mantis 0006166) and the HistoryRead evaluators (`TimeWeightedAggregator`, SqlServer + QuestDB) count Good regions only, so live AggregateFilter results disagreed with HistoryRead whenever Uncertain data was present. ScadaBridge uses neither AggregateFilter nor these aggregates. + +- The copied upstream files above are **unchanged** (their SHA-256s still match `baseline-sources.json`). +- `GoodOnlyStatusAggregateCalculator.cs` (application code, not a baseline copy) subclasses the baseline `StatusAggregateCalculator` and replaces only the PercentGood / DurationGood computation: the same slice values, simple bounds, stepped regions, empty-slice `BadNoData`, timestamps, value type and Calculated bit, with a region counted as Good only when its status is Good — whatever `TreatUncertainAsBad` says (when true, Uncertain is Bad, so it was never Good). `CompatibilityAggregateFactory.Create` routes exactly those two IDs to it; the other 35 IDs, including DurationBad / PercentBad / WorstQuality, are the baseline unchanged. +- Golden test: the 64 vectors this changes (38 PercentGood + 26 DurationGood, all with an effective `TreatUncertainAsBad = false` over data with Uncertain regions) are exempted **exactly** — the PercentGood / DurationGood vectors whose baseline output differs from the same case's baseline vector with `TreatUncertainAsBad = true`, where 1.5.378.182 already counted Good regions only — and each is pinned to that twin's output. The pin is therefore baseline evidence, never regenerated from the application's calculator; any further drift in any of the 6,144 cases fails. `Sdk2AggregateCompatibilityTests.Issue809_*` also pins a hand-computed case. +- Live-vs-history parity: `SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples` compares the registered calculator with `TimeWeightedAggregator` over Good / Uncertain / Bad mixes with no masks, and checks that the data discriminates against the pre-#809 baseline. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs index 52f74054d..5f38bb1ad 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs @@ -10,7 +10,9 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// /// The aggregate evaluators (AggregateEvaluator / TimeWeightedAggregator in /// Driver.Historian.Sql.Common) are written against fixed settings, not parameterised: Uncertain time counts as -/// Good ( = false) in the time-weighted family; the per-sample Average / Minimum / +/// Good ( = false) in TimeAverage2 / Total2, while PercentGood / DurationGood count +/// Good time only and PercentBad / DurationBad Bad time only — Uncertain in neither (Part 13 §5.4.3.31-34, Mantis +/// 0006166); the per-sample Average / Minimum / /// Maximum / Range / *ActualTime / Count are defined by Part 13 over Good values whatever this setting, as the /// SDK computes them under it; an interval is Good only when all of /// it is ( = 100) and Bad only when none of it is non-Bad @@ -23,6 +25,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// the SDK manager so live AggregateFilter subscriptions agree with history reads; a test pins both sides. /// /// +/// The live calculators are the application's frozen 1.5.378.182 copies (Compatibility/Aggregates), whose +/// ComputeDurationGoodBad counts Uncertain time as Good under = false. That +/// made live PercentGood / DurationGood disagree with HistoryRead whenever Uncertain data was present. #809 +/// (owner-approved 2026-11-11) overrides exactly those two (GoodOnlyStatusAggregateCalculator), so live and +/// HistoryRead now agree on them — pinned by +/// SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples. +/// Other known live-vs-HistoryRead differences (e.g. #821, an Uncertain first sample in a late-start interval) are +/// documented in docs/Historian.md. +/// +/// /// A request asking for UseServerCapabilitiesDefaults, or spelling out exactly these values, is served; /// any other configuration is refused per node with BadAggregateConfigurationRejected rather than silently /// answered under settings the client did not ask for (it used to be ignored). diff --git a/tests/Drivers/Historian.Conformance/QualityMixConformance.cs b/tests/Drivers/Historian.Conformance/QualityMixConformance.cs index daf57deeb..572a6b303 100644 --- a/tests/Drivers/Historian.Conformance/QualityMixConformance.cs +++ b/tests/Drivers/Historian.Conformance/QualityMixConformance.cs @@ -71,7 +71,7 @@ public abstract class QualityMixConformance : ConformanceBase /// Hand-computed Part 13 DurationBad of Q's eight intervals, seconds. private static readonly double[] QDurationBad = [3, 0, 0, 0, 5, 0, 0, 3]; - /// What the 1.5.378.182 baseline answers for Q's PercentGood (Uncertain counted as Good, #796). + /// What the untouched 1.5.378.182 baseline answers for Q's PercentGood (Uncertain counted as Good, #796; live before #809). private static readonly double[] QBaselinePercentGood = [70, 100, 100, 100, 50, 100, 100, 70]; /// @@ -182,7 +182,8 @@ public abstract class QualityMixConformance : ConformanceBase var pGood = (await src.ReadProcessedAsync("Q", At(0), At(80), Ten, HistoryAggregateType.PercentGood, Ct)).Samples; ShouldBeClose(Values(good), Values(pGood).Select(p => p * Ten.TotalMilliseconds / 100).ToArray(), "DurationGood vs PercentGood"); - // The baseline (the live AggregateFilter calculator, #809) answers Q's DurationGood with the Uncertain time added. + // The untouched baseline (the live AggregateFilter calculator before #809) answers Q's DurationGood with the + // Uncertain time added; the live calculator now answers the Part 13 values above (SdkAggregateParityTests). var baseline = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.DurationGood, At(0), At(80), Ten, Q.Select(r => (r.T, r.V, r.Status))); ShouldBeClose(baseline.Select(r => Convert.ToDouble(r.Value)).ToArray(), diff --git a/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs b/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs index b791aa83b..e1274e597 100644 --- a/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs +++ b/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs @@ -2,7 +2,7 @@ using Opc.Ua; using Opc.Ua.Server; using ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates; using BaselineAggregateCalculator = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.AggregateCalculator; -using BaselineStatusAggregateCalculator = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.StatusAggregateCalculator; +using BaselineAggregators = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.Aggregators; using Shouldly; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; @@ -18,17 +18,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance; /// /// Shared (source-linked) by Sql.Common.Tests and both SQL backends' integration suites. /// -/// PercentGood (and, since #820, DurationGood) are the aggregates not taken as-is (#796). The 1.5.378.182 baseline changed +/// PercentGood / DurationGood (#796, #820, #809). The 1.5.378.182 baseline changed /// StatusAggregateCalculator.ComputeDurationGoodBad (1.5.378.106 and earlier counted Good regions only): /// with TreatUncertainAsBad = false it counts Uncertain regions as Good time (as Bad when true). Part 13 /// keeps Uncertain out of both (§5.4.3.31 DurationGood counts Good regions; §5.4.3.33 PercentGood = /// DurationGood ÷ interval; Mantis 0006166, closed "no change required": an all-Uncertain interval is -/// PercentGood 0 and PercentBad 0), and so does ours. therefore answers PercentGood -/// (and DurationGood) through : the SAME baseline calculator, region machinery (simple -/// bounds, end-of-data 1 ms region, status and Partial bits) and server-default configuration, counting only -/// Good regions — exactly what the pre-.182 oracle computed. PercentBad / DurationBad still count Bad regions only -/// (unchanged by .182) and are compared as-is. runs the baseline untouched, for the test that pins -/// the divergence. +/// PercentGood 0 and PercentBad 0), and so does ours. Since #809 the LIVE calculators follow Part 13 too: +/// CompatibilityAggregateFactory serves those two through the application's +/// GoodOnlyStatusAggregateCalculator (the same baseline region machinery — simple bounds, end-of-data 1 ms +/// region, status and Partial bits — counting only Good regions), so / are +/// exactly what a live AggregateFilter computes, for every aggregate. PercentBad / DurationBad count Bad regions only +/// (unchanged by .182). runs the untouched baseline (what live served before #809), for the +/// tests that pin that divergence. /// /// /// Not 100 − PercentBad under TreatUncertainAsBad = true: that flag also changes the region machinery @@ -54,8 +55,9 @@ public static class SdkAggregateOracle }; /// - /// Runs the Part 13 oracle for over : the baseline calculator, - /// except PercentGood / DurationGood, which count Good regions only (see the remarks, #796/#820). + /// Runs the Part 13 oracle for over : the live calculator + /// () — the baseline, except PercentGood / DurationGood, which count Good regions only (see the + /// remarks, #796/#820/#809). /// /// The aggregate. /// Request start (later than for a reverse read). @@ -65,16 +67,29 @@ public static class SdkAggregateOracle /// The oracle's processed values, in the order it returns them. public static IReadOnlyList Run(HistoryAggregateType aggregate, DateTime start, DateTime end, TimeSpan interval, IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples) => - aggregate is HistoryAggregateType.PercentGood or HistoryAggregateType.DurationGood - ? RunCore(new LegacyCalculatorAdapter(new Part13GoodCalculator(AggregateId(aggregate), - start, end, interval.TotalMilliseconds, false, DefaultConfiguration, Telemetry, - usePercent: aggregate == HistoryAggregateType.PercentGood)), start, end, samples) - : RunSdkAsIs(aggregate, start, end, interval, samples); + RunLive(aggregate, start, end, interval, samples); /// - /// Runs the baseline (1.5.378.182) calculator for exactly as it computes it, under - /// — including its PercentGood rule (Uncertain counted as Good) that - /// deliberately does not use (#796). + /// Runs exactly the calculator the server registers for live AggregateFilter monitored items + /// (CompatibilityAggregateFactory.Create, the #809 Good-only PercentGood / DurationGood included). + /// + /// The aggregate. + /// Request start (later than for a reverse read). + /// Request end. + /// Processing interval. + /// Every stored sample of the tag, any order (sorted here). + /// The configuration (default ). + /// The calculator's processed values, in the order it returns them. + public static IReadOnlyList RunLive(HistoryAggregateType aggregate, DateTime start, DateTime end, + TimeSpan interval, IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples, + AggregateConfiguration? configuration = null) => + RunCore(CompatibilityAggregateFactory.Create(AggregateId(aggregate), start, end, interval.TotalMilliseconds, false, + configuration ?? DefaultConfiguration, Telemetry), start, end, samples); + + /// + /// Runs the untouched baseline (1.5.378.182) calculator for exactly as it computes it, + /// under — including its PercentGood / DurationGood rule (Uncertain counted as + /// Good) that neither HistoryRead nor, since #809, the live calculators use. /// /// The aggregate. /// Request start (later than for a reverse read). @@ -86,8 +101,9 @@ public static class SdkAggregateOracle public static IReadOnlyList RunSdkAsIs(HistoryAggregateType aggregate, DateTime start, DateTime end, TimeSpan interval, IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples, AggregateConfiguration? configuration = null) => - RunCore(CompatibilityAggregateFactory.Create(AggregateId(aggregate), start, end, interval.TotalMilliseconds, false, - configuration ?? DefaultConfiguration, Telemetry), start, end, samples); + RunCore(new LegacyCalculatorAdapter(BaselineAggregators.CreateStandardCalculator(AggregateId(aggregate), start, end, + interval.TotalMilliseconds, false, configuration ?? DefaultConfiguration, Telemetry) + ?? throw new ArgumentOutOfRangeException(nameof(aggregate))), start, end, samples); private static IReadOnlyList RunCore(IAggregateCalculator calc, DateTime start, DateTime end, IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples) @@ -155,42 +171,7 @@ public static class SdkAggregateOracle problems.ShouldBeEmpty($"{label} differs from the SDK:\n" + string.Join("\n", problems)); } - /// - /// The baseline with PercentGood / DurationGood counting Good - /// regions only — Part 13 §5.4.3.31/§5.4.3.33, and the 1.5.378.106 rule. Everything else (bounds, regions, Partial, - /// status, the millisecond Duration unit) is the baseline's own code, run under the same configuration as - /// PercentBad / DurationBad. - /// - private sealed class Part13GoodCalculator(NodeId aggregateId, DateTime startTime, DateTime endTime, - double processingInterval, bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry, - bool usePercent) - : BaselineStatusAggregateCalculator(aggregateId, startTime, endTime, processingInterval, stepped, configuration, telemetry) - { - protected override LegacyDataValue ComputeValue(TimeSlice slice) - { - var values = GetValuesWithSimpleBounds(slice); - if (values == null || values.Count == 0) return GetNoDataValue(slice); - - var regions = GetRegionsInValueSet(values, false, true); - double good = 0, total = 0; - foreach (var region in regions) - { - total += region.Duration; - if (StatusCode.IsGood(region.StatusCode)) good += region.Duration; - } - - var value = new LegacyDataValue - { - WrappedValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.SdkValueBoundary.FromClr(usePercent ? good / total * 100 : good), - SourceTimestamp = GetTimestamp(slice), - ServerTimestamp = GetTimestamp(slice), - }; - value.StatusCode = value.StatusCode.WithAggregateBits(AggregateBits.Calculated); - return value; - } - } - - /// The adapter, for a calculator built here. + /// The adapter, for a baseline calculator built here. private sealed class LegacyCalculatorAdapter(BaselineAggregateCalculator calculator) : IAggregateCalculator { public NodeId AggregateId => calculator.AggregateId; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs index d7e768fc2..eb25f5cd5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs @@ -414,9 +414,10 @@ public sealed class SdkAggregateParityTests public void PercentGood_keeps_Uncertain_out_of_Good_per_Part_13_where_the_baseline_SDK_counts_it_as_Good( double from, double to) { - // #796, documented divergence (SdkAggregateOracle remarks). Part 13 §5.4.3.31 / §5.4.3.33: PercentGood is the Good - // share of the interval. The 1.5.378.182 StatusAggregateCalculator, under the server's TreatUncertainAsBad = false, - // adds Uncertain regions to it (1.5.378.106 did not). Hand-computed over 4 × 5 s regions G / U / B / G: + // #796, documented divergence of the untouched baseline (SdkAggregateOracle remarks; live follows Part 13 since + // #809). Part 13 §5.4.3.31 / §5.4.3.33: PercentGood is the Good share of the interval. The 1.5.378.182 + // StatusAggregateCalculator, under the server's TreatUncertainAsBad = false, adds Uncertain regions to it + // (1.5.378.106 did not). Hand-computed over 4 × 5 s regions G / U / B / G: // Good 10 s ⇒ 50 %, Bad 5 s ⇒ 25 %, Uncertain 5 s in neither; the baseline says 75 % Good. Pct(AsIs(HistoryAggregateType.PercentGood, from, to, GoodUncertainBadGood)) .ShouldBe(75d, 1e-9, "the baseline counts the Uncertain 5 s as Good — if this moves, revisit the oracle (#796)"); @@ -524,15 +525,16 @@ public sealed class SdkAggregateParityTests public void An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data( double from, double to) { - // #821, pinned divergence (pending the #809/#821 owner decision — this test records, it does not choose). + // #821, pinned divergence (pending the #821 owner decision — this test records, it does not choose). // Part 13 §5.4.3.3/§5.4.3.8: the interpolated-bounding signal is defined from the first sample only — 10 → 11 over // [5,12) (73.5) and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s. const double part13Total = 73.5 + 88 + 32d / 13; Pct(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9); Pct(Ours(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9); - // The frozen 1.5.378.182 calculators — exactly what the server registers for live AggregateFilter items - // (CompatibilityAggregateFactory; Sdk2AggregateCompatibilityTests proves the runtime manager serves the same code) + // The frozen 1.5.378.182 calculators — for Total / TimeAverage exactly what the server registers for live + // AggregateFilter items (CompatibilityAggregateFactory; Sdk2AggregateCompatibilityTests proves the runtime manager + // serves the same code) // — integrate the uncovered [0,5) as a ramp 0 → 10 (+25) and divide TimeAverage by the WHOLE 20 s interval. Pct(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart)).ShouldBe(part13Total + 25, 1e-9); Pct(AsIs(HistoryAggregateType.TimeAverage, from, to, UncertainFirstLateStart)).ShouldBe((part13Total + 25) / 20, 1e-9); @@ -557,16 +559,108 @@ public sealed class SdkAggregateParityTests Stock201(agg, from, to, GoodFirstLateStart).Single().Value!.Value.ShouldBe(expected, 1e-9); } - // #809 on the same input: DurationGood / PercentGood — Part 13 (ours) and stock SDK 2.0.1 count Good regions only - // ([12,20) = 8 s); the frozen live calculators also count the Uncertain [5,12) as Good (15 s). + // #809 on the same input: DurationGood / PercentGood — Part 13 (ours), stock SDK 2.0.1 and, since #809, the live + // calculators count Good regions only ([12,20) = 8 s); the untouched baseline (live before #809) also counted the + // Uncertain [5,12) as Good (15 s). Pct(Ours(HistoryAggregateType.DurationGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(8_000d, 1e-9); Stock201(HistoryAggregateType.DurationGood, from, to, UncertainFirstLateStart).Single().Value!.Value.ShouldBe(8_000d, 1e-9); + Pct(Sdk(HistoryAggregateType.DurationGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(8_000d, 1e-9); Pct(AsIs(HistoryAggregateType.DurationGood, from, to, UncertainFirstLateStart)).ShouldBe(15_000d, 1e-9); Pct(Ours(HistoryAggregateType.PercentGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(40d, 1e-9); Stock201(HistoryAggregateType.PercentGood, from, to, UncertainFirstLateStart).Single().Value!.Value.ShouldBe(40d, 1e-9); + Pct(Sdk(HistoryAggregateType.PercentGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(40d, 1e-9); Pct(AsIs(HistoryAggregateType.PercentGood, from, to, UncertainFirstLateStart)).ShouldBe(75d, 1e-9); } + /// + /// #809: the #810 QualityMix tag Q — Good / Uncertain / Bad samples and an outage marker, Uncertain regions starting + /// at a sample, through a sloped bound and held into an outage, with Good samples before (−10) and after (+90) every + /// window so no interval holds the start or the end of the data. + /// + private static readonly StoredSample[] UncertainMix = + [ + S(-10, HistorianValue.FromDouble(10), Good), + S(0, HistorianValue.FromDouble(10), Good), + S(4, HistorianValue.FromDouble(12), Uncertain), + S(7, HistorianValue.FromDouble(13), Bad), + S(10, HistorianValue.FromDouble(14), Good), + S(13, HistorianValue.FromDouble(15), Uncertain), + S(20, HistorianValue.FromDouble(16), Uncertain), + S(25, HistorianValue.FromDouble(17), Uncertain), + S(33, HistorianValue.FromDouble(18), Uncertain), + S(40, HistorianValue.FromDouble(20), Good), + S(45, HistorianValue.Empty, BadNoCommunication), + S(50, HistorianValue.FromDouble(22), Uncertain), + S(56, HistorianValue.FromDouble(23), Good), + S(64, HistorianValue.FromDouble(25), Uncertain), + S(67, HistorianValue.FromDouble(26), Good), + S(75, HistorianValue.FromDouble(27), Bad), + S(78, HistorianValue.FromDouble(28), Good), + S(90, HistorianValue.FromDouble(30), Good), + ]; + + /// Windows of whole intervals only (no truncated interval — ours flags it Partial, a documented divergence). + public static TheoryData WholeIntervalWindows => new() + { + { 0, 80, 20 }, { 80, 0, 20 }, + { 0, 80, 10 }, { 80, 0, 10 }, + { 5, 80, 15 }, { 80, 5, 15 }, + { 0, 60, 0 }, { 60, 0, 0 }, + }; + + [Theory] + [MemberData(nameof(WholeIntervalWindows))] + public void Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples( + double from, double to, double intervalS) + { + // #809: a live AggregateFilter monitored item and a HistoryRead-Processed (SqlServer / QuestDB share + // TimeWeightedAggregator) answer PercentGood / DurationGood identically over Uncertain data — same values, + // timestamps, status and Partial / Calculated bits, with NO documented-divergence mask. RunLive is exactly + // CompatibilityAggregateFactory.Create, the calculator the server registers (Sdk2AggregateCompatibilityTests + // proves the booted runtime manager serves the same code). + var interval = intervalS > 0 ? TimeSpan.FromSeconds(intervalS) : (At(to) - At(from)).Duration(); + var failures = new List(); + var discriminated = 0; + foreach (var agg in new[] { HistoryAggregateType.PercentGood, HistoryAggregateType.DurationGood }) + foreach (var (name, data) in new[] { ("UncertainMix", UncertainMix), ("DenseUncertain", DenseUncertain) }) + { + var rows = data.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray(); + var live = SdkAggregateOracle.RunLive(agg, At(from), At(to), interval, rows); + try + { + SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), live, $"live {agg} {name} {from}→{to}/{intervalS}s"); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + // The data must discriminate: the pre-#809 live calculator (the untouched baseline) counts the Uncertain + // time as Good, so it disagrees somewhere — or this parity would pass against the old behaviour too. + var before = SdkAggregateOracle.RunSdkAsIs(agg, At(from), At(to), interval, rows); + if (before.Zip(live).Any(p => Math.Abs(Convert.ToDouble(p.First.Value) - Convert.ToDouble(p.Second.Value)) > 1e-9)) + discriminated++; + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + discriminated.ShouldBe(4, "every aggregate × data set must differ from the pre-#809 live answer somewhere"); + } + + [Fact] + public void Live_AggregateFilter_PercentGood_and_DurationGood_are_the_hand_computed_Part_13_values() + { + // #809, hand-computed over UncertainMix's eight 10 s intervals (QualityMixConformance derives them region by + // region): Good seconds 4, 3, 0, 0, 5, 4, 3, 2. The pre-#809 live answer added the Uncertain time + // (7, 10, 10, 10, 5, 10, 10, 7 s). + double[] goodSeconds = [4, 3, 0, 0, 5, 4, 3, 2]; + var rows = UncertainMix.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray(); + var ten = TimeSpan.FromSeconds(10); + SdkAggregateOracle.RunLive(HistoryAggregateType.DurationGood, At(0), At(80), ten, rows) + .Select(r => Convert.ToDouble(r.Value)).ShouldBe(goodSeconds.Select(g => g * 1000), tolerance: 1e-9); + SdkAggregateOracle.RunLive(HistoryAggregateType.PercentGood, At(0), At(80), ten, rows) + .Select(r => Convert.ToDouble(r.Value)).ShouldBe(goodSeconds.Select(g => g * 10), tolerance: 1e-9); + SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.DurationGood, At(0), At(80), ten, rows) + .Select(r => Convert.ToDouble(r.Value)).ShouldBe([7000d, 10000, 10000, 10000, 5000, 10000, 10000, 7000], tolerance: 1e-9); + } + /// #684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6). private static readonly StoredSample[] UncertainBeyond = [ diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs index 6dffd4484..7132eaf93 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs @@ -6,9 +6,10 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// #821 / #809 facts, through the BOOTED server's runtime AggregateManager — the calculators a live -/// AggregateFilter monitored item gets, under the server's own default configuration. This pins current behaviour for -/// the owner decision; it does not choose. The HistoryRead (Part 13) answers and SDK 2.0.1's stock answers for the -/// same input are pinned beside it in SdkAggregateParityTests. +/// AggregateFilter monitored item gets, under the server's own default configuration. The #821 Total / TimeAverage +/// facts pin current behaviour for the owner decision; they do not choose. #809 is decided (2026-11-11): live +/// PercentGood / DurationGood count Good time only, equal to HistoryRead. The HistoryRead (Part 13) answers and SDK +/// 2.0.1's stock answers for the same input are pinned beside it in SdkAggregateParityTests. /// public sealed class LiveAggregateLateStartFactsTests { @@ -45,9 +46,10 @@ public sealed class LiveAggregateLateStartFactsTests const double part13Total = 73.5 + 88 + 32d / 13; One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total + 25, 1e-9); // 188.96… One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe((part13Total + 25) / 20, 1e-9); // 9.448… (Part 13: 10.93…) - // #809: the Uncertain [5,12) counts as Good live (HistoryRead: 8 000 ms / 40 %). - One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(15_000d, 1e-9); - One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(75d, 1e-9); + // #809: the Uncertain [5,12) is not Good — only [12,20) is, live as in HistoryRead (8 000 ms / 40 %; before #809 + // the live calculator counted the Uncertain time too: 15 000 ms / 75 %). + One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(8_000d, 1e-9); + One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(40d, 1e-9); // DurationBad / PercentBad agree with HistoryRead: the 5 s before the data is Bad. One(ObjectIds.AggregateFunction_DurationBad).ShouldBe(5_000d, 1e-9); One(ObjectIds.AggregateFunction_PercentBad).ShouldBe(25d, 1e-9); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs index 87d1fd57e..e40367fae 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs @@ -9,8 +9,20 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// Compares all preserved aggregate factories with independently generated SDK 1.5.378.182 vectors. +/// +/// #809 (owner-approved 2026-11-11): PercentGood / DurationGood no longer count Uncertain time as Good. The baseline +/// vectors that changes are exempted EXACTLY — a PercentGood / DurationGood vector whose output differs from the same +/// case computed by the baseline SDK with TreatUncertainAsBad = true (its "twin": there 1.5.378.182 already +/// counted Good regions only) — and each is pinned to that twin instead. The twin is baseline evidence, not output of +/// the application's calculator, so the pin never regenerates expectations from the implementation. +/// public sealed class Sdk2AggregateCompatibilityTests { + private const string PercentGood = "AggregateFunction_PercentGood"; + private const string DurationGood = "AggregateFunction_DurationGood"; + private const int Issue809PercentGoodExemptions = 38; + private const int Issue809DurationGoodExemptions = 26; + [Theory] [InlineData(false)] [InlineData(true)] @@ -19,57 +31,145 @@ public sealed class Sdk2AggregateCompatibilityTests await using var rig = throughRuntimeManager ? await ServerEventRig.StartAsync("AggregateBaseline", encrypted: false, TestContext.Current.CancellationToken) : null; - var baseline = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; - var telemetry = DefaultTelemetry.Create(_ => { }); - var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc); + var baseline = LoadBaseline(); + var exemptions = Issue809Exemptions(baseline); var failures = new List(); foreach (var vector in baseline.Vectors) { - var id = (NodeId)typeof(ObjectIds).GetProperty(vector.Aggregate)!.GetValue(null)!; - var start = epoch.AddSeconds(vector.Reverse ? 70 : 0); var end = epoch.AddSeconds(vector.Reverse ? 0 : 70); - var profile = vector.Configuration; - var config = new AggregateConfiguration - { - TreatUncertainAsBad = profile?.TreatUncertainAsBad ?? vector.UncertainAsBad, - PercentDataBad = profile?.PercentDataBad ?? 100, PercentDataGood = profile?.PercentDataGood ?? 100, - UseSlopedExtrapolation = profile?.UseSlopedExtrapolation ?? false, - UseServerCapabilitiesDefaults = profile?.UseServerCapabilitiesDefaults ?? false, - }; - var calculator = throughRuntimeManager - ? rig!.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config) - : CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry); - calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}."); - var output = new List(); var accepted = new List(); string? error = null; - try - { - var samples = baseline.Data[vector.DataSet].OrderBy(x => x.Seconds); - foreach (var sample in vector.Reverse ? samples.Reverse() : samples) - { - var time = epoch.AddSeconds(sample.Seconds); - accepted.Add(calculator.QueueRawValue(new DataValue(SdkValueBoundary.FromClr(Payload(sample)), sample.Status, time, time.AddTicks(sample.ServerOffsetTicks), sample.SourcePicoseconds, sample.ServerPicoseconds))); - if (vector.Incremental) Drain(false); - } - Drain(true); - } - catch (Exception ex) { error = ex.GetType().FullName; } - if (error != vector.Exception || !output.SequenceEqual(vector.Output) || (vector.QueueAccepted is { } expectedAccepted && !accepted.SequenceEqual(expectedAccepted))) - { - failures.Add($"{vector.Aggregate}/{vector.DataSet}/rev={vector.Reverse}/step={vector.Stepped}/uncertainBad={vector.UncertainAsBad}/incremental={vector.Incremental}: error {vector.Exception} vs {error}; expected {JsonSerializer.Serialize(vector.Output)} actual {JsonSerializer.Serialize(output)}"); - } - void Drain(bool partial) - { - for (var guard = 0; guard < 100; guard++) - { - if (!calculator.TryGetProcessedValue(partial, out var value)) return; - var payload = SdkValueBoundary.ToClr(value.WrappedValue); - output.Add(new(partial, payload?.GetType().Name, Format(payload), value.StatusCode.Code, LegacyTimestampTicks(value.SourceTimestamp), LegacyTimestampTicks(value.ServerTimestamp), value.SourcePicoseconds, value.ServerPicoseconds)); - } - throw new InvalidOperationException("Baseline calculator did not terminate."); - } + // #809: an exempted case is held to its baseline uncertain-as-bad twin; every other case to its own vector. + var expected = exemptions.TryGetValue(vector, out var twin) ? twin.Output : vector.Output; + if (Check(baseline, vector, expected, rig) is { } failure) failures.Add(failure); } baseline.Vectors.Length.ShouldBe(6144); baseline.Vectors.Select(x => x.Aggregate).Distinct().Count().ShouldBe(37); - failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing."); + exemptions.Keys.Count(x => x.Aggregate == PercentGood).ShouldBe(Issue809PercentGoodExemptions); + exemptions.Keys.Count(x => x.Aggregate == DurationGood).ShouldBe(Issue809DurationGoodExemptions); + failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing (#809 exemptions: the uncertain-as-bad twin)."); + } + + /// + /// #809 pin: the exemptions are exactly the PercentGood / DurationGood cases that counted Uncertain as Good, and + /// every PercentGood / DurationGood case — exempt or not, whatever its configuration — now answers its baseline + /// uncertain-as-bad twin (Good regions only). + /// + [Fact] + public void Issue809_percent_and_duration_good_count_only_good_time() + { + var baseline = LoadBaseline(); + var exemptions = Issue809Exemptions(baseline); + var failures = new List(); + var checkedCount = 0; + var twinned = 0; + foreach (var vector in baseline.Vectors.Where(v => v.Aggregate is PercentGood or DurationGood)) + { + // A data set with no uncertain-as-bad twin (the numeric-width edge sets) has no Uncertain region to move. + var twin = UncertainAsBadTwin(baseline, vector); + if (exemptions.ContainsKey(vector)) + EffectiveTreatUncertainAsBad(vector).ShouldBeFalse($"{Describe(vector)}: only an Uncertain-as-Good case may change."); + if (Check(baseline, vector, twin?.Output ?? vector.Output, rig: null) is { } failure) failures.Add(failure); + checkedCount++; + if (twin is not null) twinned++; + } + checkedCount.ShouldBe(352); + twinned.ShouldBeGreaterThan(exemptions.Count); + exemptions.Count.ShouldBe(Issue809PercentGoodExemptions + Issue809DurationGoodExemptions); + failures.ShouldBeEmpty("PercentGood / DurationGood must count Good regions only (Part 13 §5.4.3.31/§5.4.3.33, #809)."); + } + + /// + /// #809 hand-computed pin, independent of the expected outputs in any fixture: the "uncertain" data set (0 s Good, + /// 5 s Uncertain, 15 s Good, 25 s Bad, 35 s Good, 50 s Good) over 20 s intervals from 0 to 70 s, sloped, forward. + /// [0,20): Good 0–5 and 15–20 = 10 s (the baseline said 20 s); [20,40): the bound at 20 slopes into the Bad sample + /// (Uncertain), 25–35 Bad, 35–40 Good = 5 s (baseline 10 s); [40,60): 20 s; [60,70): all 10 s of the partial + /// interval = 100 %. + /// + [Theory] + [InlineData(DurationGood, new[] { 10000d, 5000d, 20000d, 10000d })] + [InlineData(PercentGood, new[] { 50d, 25d, 100d, 100d })] + public void Issue809_uncertain_time_is_not_good_hand_computed(string aggregate, double[] expected) + { + var baseline = LoadBaseline(); + var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped + && !v.UncertainAsBad && !v.Incremental && v.Configuration is null); + var output = Run(baseline, vector, rig: null, out var error); + error.ShouldBeNull(); + output.ShouldAllBe(x => x.Type == "Double"); + output.Select(x => double.Parse(x.Payload!, CultureInfo.InvariantCulture)).ShouldBe(expected); + } + + private static Baseline LoadBaseline() => + JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; + + private static bool EffectiveTreatUncertainAsBad(Vector vector) => vector.Configuration?.TreatUncertainAsBad ?? vector.UncertainAsBad; + + /// The same case computed by the baseline SDK with TreatUncertainAsBad = true and no explicit profile. + private static Vector? UncertainAsBadTwin(Baseline baseline, Vector vector) => + baseline.Vectors.SingleOrDefault(v => v.Aggregate == vector.Aggregate && v.DataSet == vector.DataSet && v.Reverse == vector.Reverse + && v.Stepped == vector.Stepped && v.Incremental == vector.Incremental && v.UncertainAsBad && v.Configuration is null); + + /// #809: the PercentGood / DurationGood vectors whose baseline output differs from their uncertain-as-bad twin. + private static Dictionary Issue809Exemptions(Baseline baseline) => + baseline.Vectors + .Where(v => v.Aggregate is PercentGood or DurationGood) + .Select(v => (Vector: v, Twin: UncertainAsBadTwin(baseline, v))) + .Where(x => x.Twin is not null && !x.Vector.Output.SequenceEqual(x.Twin.Output)) + .ToDictionary<(Vector Vector, Vector? Twin), Vector, Vector>(x => x.Vector, x => x.Twin!, System.Collections.Generic.ReferenceEqualityComparer.Instance); + + private static string Describe(Vector vector) => + $"{vector.Aggregate}/{vector.DataSet}/rev={vector.Reverse}/step={vector.Stepped}/uncertainBad={vector.UncertainAsBad}/incremental={vector.Incremental}/config={JsonSerializer.Serialize(vector.Configuration)}"; + + private static string? Check(Baseline baseline, Vector vector, Value[] expected, ServerEventRig? rig) + { + var output = Run(baseline, vector, rig, out var error); + return error != vector.Exception || !output.SequenceEqual(expected) || (vector.QueueAccepted is { } expectedAccepted && !output.Accepted.SequenceEqual(expectedAccepted)) + ? $"{Describe(vector)}: error {vector.Exception} vs {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(output.ToArray())}" + : null; + } + + private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error) + { + var telemetry = DefaultTelemetry.Create(_ => { }); + var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc); + var id = (NodeId)typeof(ObjectIds).GetProperty(vector.Aggregate)!.GetValue(null)!; + var start = epoch.AddSeconds(vector.Reverse ? 70 : 0); var end = epoch.AddSeconds(vector.Reverse ? 0 : 70); + var profile = vector.Configuration; + var config = new AggregateConfiguration + { + TreatUncertainAsBad = profile?.TreatUncertainAsBad ?? vector.UncertainAsBad, + PercentDataBad = profile?.PercentDataBad ?? 100, PercentDataGood = profile?.PercentDataGood ?? 100, + UseSlopedExtrapolation = profile?.UseSlopedExtrapolation ?? false, + UseServerCapabilitiesDefaults = profile?.UseServerCapabilitiesDefaults ?? false, + }; + var calculator = rig is not null + ? rig.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config) + : CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry); + calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}."); + var output = new RunOutput(); + error = null; + try + { + var samples = baseline.Data[vector.DataSet].OrderBy(x => x.Seconds); + foreach (var sample in vector.Reverse ? samples.Reverse() : samples) + { + var time = epoch.AddSeconds(sample.Seconds); + output.Accepted.Add(calculator.QueueRawValue(new DataValue(SdkValueBoundary.FromClr(Payload(sample)), sample.Status, time, time.AddTicks(sample.ServerOffsetTicks), sample.SourcePicoseconds, sample.ServerPicoseconds))); + if (vector.Incremental) Drain(false); + } + Drain(true); + } + catch (Exception ex) { error = ex.GetType().FullName; } + return output; + + void Drain(bool partial) + { + for (var guard = 0; guard < 100; guard++) + { + if (!calculator.TryGetProcessedValue(partial, out var value)) return; + var payload = SdkValueBoundary.ToClr(value.WrappedValue); + output.Add(new(partial, payload?.GetType().Name, Format(payload), value.StatusCode.Code, LegacyTimestampTicks(value.SourceTimestamp), LegacyTimestampTicks(value.ServerTimestamp), value.SourcePicoseconds, value.ServerPicoseconds)); + } + throw new InvalidOperationException("Baseline calculator did not terminate."); + } } // UA FILETIME zero is the old CLR unset sentinel, not a different historical instant. @@ -84,6 +184,7 @@ public sealed class Sdk2AggregateCompatibilityTests "Single" => float.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Double" => double.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Int64" => long.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Boolean" => bool.Parse(sample.Payload!), _ => null }; static string? Format(object? value) => value switch { null => null, double number => number.ToString("R", CultureInfo.InvariantCulture), float number => number.ToString("R", CultureInfo.InvariantCulture), IFormattable f => f.ToString(null, CultureInfo.InvariantCulture), _ => value.ToString() }; + private sealed class RunOutput : List { public List Accepted { get; } = []; } record Baseline(long EpochTicks, int IntervalMilliseconds, Dictionary Data, Vector[] Vectors); record Sample(double Seconds,string Type,string? Payload,uint Status,ushort SourcePicoseconds = 0,ushort ServerPicoseconds = 0,long ServerOffsetTicks = 0); record Value(bool PartialRequested,string? Type,string? Payload,uint Status,long SourceTicks,long ServerTicks,ushort SourcePicoseconds,ushort ServerPicoseconds); From 93d18094847caaf6f555410a99249ab80049c82d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:10:59 -0500 Subject: [PATCH 05/28] =?UTF-8?q?docs(script-runner):=20=C2=A78.2=20worker?= =?UTF-8?q?=20idle=20RSS=20budget=20revised=20to=20200=20MB=20+=20scripts?= =?UTF-8?q?=20(#789)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner revised the per-shard worker idle RSS budget from 80 MB to 200 MB + scripts on 2026-11-11: compiles run in the worker, so Roslyn's code and metadata stay resident (measured 159-187 MB) and no GC setting moves it. The perf harness now reports both RSS rows against the 200 MB constant with a PASS/MISS mark; A-3 records what the measurement showed. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- ...1-09-out-of-process-script-runner-design.md | 4 ++-- .../ScriptRunnerPerfTests.cs | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/plans/2026-11-09-out-of-process-script-runner-design.md b/docs/plans/2026-11-09-out-of-process-script-runner-design.md index 234aefca1..b48d3db87 100644 --- a/docs/plans/2026-11-09-out-of-process-script-runner-design.md +++ b/docs/plans/2026-11-09-out-of-process-script-runner-design.md @@ -536,7 +536,7 @@ size for 2× headroom (10k/s). The passthrough fast path is excluded because it | Host CPU for IPC | ≤ 25% of one core at 10k evals/s | Serialisation + pipe I/O, measured by an opt-in perf test. Since lmxopcua#789 the harness reports it as the out-of-process runner's host CPU minus the in-process runner's in the same phase ("vs InProcess"): the raw figure includes the harness's own pacing loop and pool, which InProcess pays too (§12.2) | | Bytes per eval on the wire | ≤ 64 B + 24 B per input | Positional slots, §4.4 | | Shard restart → first result, 1,000 cached scripts | ≤ 2 s | §7 lazy PE load | -| Worker idle RSS | ≤ 80 MB + scripts **[A-3]** | Per shard. **Proposed revision (lmxopcua#789, §12.2):** ≤ 80 MB + scripts for a worker that has only loaded cached PEs (measured 36–58 MB); ≤ 200 MB + scripts once Roslyn has run in it (measured 159–187 MB), because compiling in the worker (§14 #7) keeps Roslyn's code and metadata resident and no GC setting moves it | +| Worker idle RSS | ≤ 200 MB + scripts | Per shard. **Revised by the owner 2026-11-11 (lmxopcua#789)** from the original ≤ 80 MB + scripts **[A-3]**. Compiling in the worker (§14 #7) keeps Roslyn's code and metadata resident once it has run (measured 159–187 MB), and no GC setting moves it (§12.2). A worker that has only loaded cached PEs sits far below that (measured 36–58 MB). The perf harness reports both against 200 MB | | Kill latency after `T+G` | ≤ 100 ms | Host timer resolution plus `Process.Kill` | **Comparison with today:** the in-process cost per compiled run is one `Task.Run` hop plus `WaitAsync`, which @@ -1367,7 +1367,7 @@ Each has a recommendation, marked ★. |---|---|---| | A-1 | Unbounded recursion in a script throws `StackOverflowException` and kills the host today | Reproduce in a throwaway process (not the test host) | | A-2 | A collectible ALC whose code is still executing on an orphan thread cannot be unloaded | .NET docs / a `WeakReference` test with a spinning orphan | -| A-3 | A minimal .NET 10 worker idles at tens of MB RSS | Measure the P1 worker | +| A-3 | A minimal .NET 10 worker idles at tens of MB RSS | Measured (§12.1/§12.2): true only for a worker that has never compiled (36–58 MB); a worker that has run Roslyn idles at 159–187 MB, so §8.2 was revised to 200 MB | | A-4 | `P = max(2, cores/K)` evaluation threads per worker is enough | P1 perf harness | | A-5 | Passthrough scripts are a large share of evaluations on real deployments | Count `PassthroughScript.TryMatch` hits against compiled runs on QA (add a counter in P0) | | A-6 | A local pipe round trip costs tens of µs | P1 perf harness | diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs index 2235faa5b..f78053f6b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs @@ -32,6 +32,12 @@ public sealed class ScriptRunnerPerfTests { private const string OptInVariable = "OTOPCUA_SCRIPT_PERF"; private const int AcceptedGateEvalsPerSecond = 5_000; + + /// + /// §8.2 "Worker idle RSS ≤ 200 MB + scripts", per shard (revised by the owner 2026-11-11, lmxopcua#789: compiles + /// run in the worker, so Roslyn stays resident; the original 80 MB [A-3] is only met by a warm-PE-cache worker). + /// + private const int WorkerIdleRssBudgetMb = 200; private const int Keys = 2_000; private const int DistinctSources = 50; @@ -481,7 +487,7 @@ public sealed class ScriptRunnerPerfTests } /// - /// §8.2 "Shard restart → first result, 1,000 cached scripts ≤ 2 s", "Worker idle RSS ≤ 80 MB + scripts" and + /// §8.2 "Shard restart → first result, 1,000 cached scripts ≤ 2 s", "Worker idle RSS ≤ 200 MB + scripts" and /// "Kill latency after T+G ≤ 100 ms". Report-only. /// [Fact] @@ -559,14 +565,20 @@ public sealed class ScriptRunnerPerfTests } } + static string RssCell(long bytes) + { + var mb = bytes / 1048576.0; + return $"{mb:N0} MB {(mb <= WorkerIdleRssBudgetMb ? "PASS" : "MISS")}"; + } + report.AppendLine("| Measure | Result | §8.2 budget |") .AppendLine("|---|---:|---|") .AppendLine($"| {scripts} scripts, empty cache, all first results (compile + write) | {cold.TotalMilliseconds:N0} ms | — |") .AppendLine($"| {scripts} scripts, fresh runner on a warm cache, all first results | {warmFresh.TotalMilliseconds:N0} ms | ≤ 2,000 ms |") .AppendLine($"| kill both shards → every script's next correct result | {killRestart.TotalMilliseconds:N0} ms | ≤ 2,000 ms |") - .AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |") + .AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {RssCell(rssIdle)} | ≤ {WorkerIdleRssBudgetMb} MB + scripts |") .AppendLine($"| worker RSS with {scripts / 2} scripts compiled per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |") - .AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {rssWarmIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |") + .AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {RssCell(rssWarmIdle)} | ≤ {WorkerIdleRssBudgetMb} MB + scripts |") .AppendLine($"| worker RSS with {scripts / 2} scripts loaded from the cache per shard (max) | {rssWarmLoaded / 1048576.0:N0} MB | — |") .AppendLine($"| kill latency after T+G (5 runs, ms) | {string.Join(" / ", killLatencies.Select(l => l.ToString("N0")))} | ≤ 100 ms |"); Report(report); From fdc9473819d7198849dfa4451ca789f3a365a74c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:13:55 -0500 Subject: [PATCH 06/28] fix(twincat): Test Connect probe no longer dials the AmsNetId as an IP (#863) TwinCATDriverProbe ran a first-phase TCP check against the AmsNetId's first four octets on the AMS port (801/851). An AmsNetId is not an IP and the AMS port is not a TCP port, so Test Connect timed out against every real runtime (Z35474: dialled 5.118.56.93, an unrelated public host). The probe now opens no socket of its own: it takes the driver's path, AdsClient.Connect + ReadStateAsync through the local AMS router, bounded by the probe budget (Task.Run + WaitAsync), and classifies failures as no local AMS router / no route or target unreachable / timeout / other. The degrade-to-green 'TCP reachability only' outcome is gone - with no TCP check nothing was contacted, so a missing router is reported red. Tests: the regression test listens on the AmsNetId-derived endpoint and asserts no connection arrives (fails on the old probe); classification and budget tests drive the handshake through an internal seam. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .claude/rules/drivers.md | 4 + docs/drivers/TestConnectProbes.md | 41 ++- .../TwinCATDriverProbe.cs | 281 ++++++++-------- .../TwinCATDriverProbeTests.cs | 306 ++++++++---------- 4 files changed, 298 insertions(+), 334 deletions(-) diff --git a/.claude/rules/drivers.md b/.claude/rules/drivers.md index abe6ffff9..48167caef 100644 --- a/.claude/rules/drivers.md +++ b/.claude/rules/drivers.md @@ -268,6 +268,10 @@ host logger factory (wired in `DriverFactoryBootstrap`; the FOCAS and TwinCAT fa `DriverHealthChanged.Diagnostics` / proto `has_diagnostics`+`diagnostics`; its digest is in the publish fingerprint). - **TwinCAT (#637):** TIME/TOD publish Float64 ms, DATE/DT a UTC DateTime; a tag authored `UDInt` keeps the raw counter (and keeps historizing — DateTime is not historized). +- **TwinCAT Test Connect (#863):** the probe opens no socket of its own — `AdsClient.Connect` + `ReadStateAsync` + through the LOCAL AMS router, exactly the driver's path. Never derive a host from an AMS Net ID (it is not an IP; + the retired pre-check dialled its first four octets on the AMS port — a public stranger — and always timed out). + No router ⇒ red "No local AMS router" (no degrade-to-green). `docs/drivers/TestConnectProbes.md`. - **TwinCAT (#645, #310/#314):** opt-in ADS sum-read (`useSumRead`, `sumReadMaxItems` default 200 / ceiling 500, 32 KiB payload budget). It reads raw bytes and decodes as the declared type, so only a scalar passing the strict width+kind check (`CheckNotification`) rides a batch; value-converted pairs (DInt over INT, Real over LREAL/DINT) read singly. Only overall diff --git a/docs/drivers/TestConnectProbes.md b/docs/drivers/TestConnectProbes.md index 946303481..71e8f73d7 100644 --- a/docs/drivers/TestConnectProbes.md +++ b/docs/drivers/TestConnectProbes.md @@ -46,7 +46,7 @@ with a human-readable explanation rather than a false-green TCP-open tick. | **S7** | `Plc.OpenAsync` (COTP CR/CC + S7 setup-communication), check `IsConnected`, then `Close`. Wrong rack/slot or a non-S7 server causes `OpenAsync` to throw → handshake fail. | `"S7 connected (CPU …)"` | `10.100.0.35:1102` (python-snap7 sim) | | **AbCip** | `libplctag` Tag `InitializeAsync` (EIP session + CIP Forward Open). A CIP-level error such as tag-not-found still proves the controller answered CIP → `Ok`. A session/ForwardOpen/connect error → handshake fail. | `"CIP session OK"` | `10.100.0.35:44818` (CIP sim) | | **AbLegacy** | Same `libplctag` `InitializeAsync` handshake as AbCip, PCCC protocol family. | `"CIP session OK"` (PCCC family) | Deferred — no PLC5/SLC sim | -| **TwinCAT** | `AdsClient.Connect` + `ReadStateAsync`. See [degrade semantics](#twincat-degrade) below. | `"ADS state: {state}"` | Deferred — no ADS target | +| **TwinCAT** | `AdsClient.Connect` + `ReadStateAsync` through the **local AMS router** — no TCP pre-check. See [TwinCAT: no TCP pre-check](#twincat-no-tcp-pre-check) below. | `"ADS state: {state}"` | Z35474 (production TC3 press, read-only, via an in-process TcpRouter) | | **FOCAS** | `cnc_allclibhndl3` via a direct `DllImport("fwlib32")` in the probe. See [degrade semantics](#focas-degrade) below. | `"FOCAS handle OK"` | Deferred — no CNC + FWLIB | | **Galaxy** | gRPC unary call to `GalaxyRepository.TestConnection` on the configured mxaccessgw endpoint. See [auth-rejection rule](#galaxy-auth-rejection) below. | `"gateway gRPC OK"` | `http://10.100.0.48:5120` (mxaccessgw) | | **Heidenhain** | LSV2 `A_LG INSPECT` (password resolved from its `secret:`/`env:` reference) + `R_VR`, through the same read-only allowlist as the driver; a refusal or malformed version record → fail. No logout — the socket close ends the session. | `"LSV2 INSPECT session OK at host:port: NC "` | Deferred — no LSV2 control; simulator-proven | @@ -61,29 +61,36 @@ backend is now the external HistorianGateway (a gRPC client package, not a probe ## Degrade semantics -Three drivers have environmental constraints that can prevent the handshake +Some drivers have environmental constraints that can prevent the handshake from running on certain hosts. The **degradation principle** is: the probe must never produce a result *worse* than today's TCP-only probe. A genuine protocol rejection from a reachable device is a correct RED; an inability to -*run* the handshake at all (no FWLIB, no managed router) degrades to the -existing TCP-reachability message — still a green tick but annotated. +*run* the handshake at all (no FWLIB) degrades to the +existing TCP-reachability message — still a green tick but annotated. TwinCAT +no longer degrades (see below). -### TwinCAT degrade +### TwinCAT: no TCP pre-check -Where the handshake is available: +TwinCAT is the exception to the "TCP connect fails" row above: it opens **no socket of its own** +(#863). The driver has no IP field — an AMS Net ID is a Beckhoff identifier the AMS router maps to a +host through its route table, and the AMS port (801/851) is not a TCP port. The probe therefore takes +exactly the driver's path (`AdsTwinCATClient.ConnectAsync`): `AdsClient.Connect(netId, port)` through +the local AMS router (TwinCAT XAR, or `Beckhoff.TwinCAT.Ads.TcpRouter` hosted in-process), then one +read-only `ReadStateAsync`, all inside the probe budget. Until #863 a first-phase check dialled the Net +ID's first four octets as an IPv4 address on the AMS port — for Z35474 (`5.118.56.93.1.1`) an +unrelated public host — so Test Connect timed out against every real runtime. -- `AdsClient.Connect(netId, port)` + `ReadStateAsync` → `Ok=true`, - `"ADS state: {state}"` (Run / Config / Stop). -- An ADS **route-table rejection** from a reachable ADS router is a **true RED**: - `"Reachable at {host}:{port} but ADS handshake failed: {detail} — check the - target's ADS route table authorizes this host"`. This is the correct result: - the driver would also be unable to function without an authorized route. +| Outcome | `Ok` | Message | +|---------|------|---------| +| ReadState succeeds | `true` | `"ADS state: {state}"` (Run / Config / Stop) | +| No local AMS router (Beckhoff "Check for a running TwinCAT router instance!", `ClientPortNotOpen`, router not active, ADS client cannot load) | `false` | `"No local AMS router on this host — start TwinCAT XAR or host the Beckhoff TcpRouter; the target was not contacted (…)"` | +| Router answered but cannot reach the target (`TargetMachineNotFound`, `TargetPortNotFound`, port/access refused, host unreachable) | `false` | `"No route to {netId}:{port} or target unreachable: {code} — check the local AMS router has a route …"` | +| Sync/device timeout code, or the budget expired | `false` | `"Probe timed out after {n}s waiting for ADS state from {netId}:{port} …"` | +| Any other ADS error | `false` | `"ADS handshake with {netId}:{port} failed: {code}"` | -Where the handshake is unavailable (headless server, no TwinCAT runtime, the -managed AMS router cannot start): - -- Probe degrades to TCP-reachability: `Ok=true`, - `"(ADS handshake unavailable on this host — TCP reachability only)"`. +There is **no degrade-to-green** any more: with no TCP check, a host that cannot run the handshake +has contacted nothing, so it reports the missing router rather than a green tick. A stock admin node +has no AMS router, so TwinCAT Test Connect is red there by design — the driver needs the same router. ### FOCAS degrade diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriverProbe.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriverProbe.cs index 8300a2b46..f1e2933e8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriverProbe.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriverProbe.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Net.Sockets; using System.Text.Json; using System.Text.Json.Serialization; using TwinCAT.Ads; @@ -8,59 +7,40 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; /// -/// Two-phase, degrade-guarded Test-Connect probe for the -shaped -/// driver config. Phase 1: a bare TCP connect to the first device's AMS router host (first four -/// octets of the AMS Net ID) on the AMS port — fast rejection of unreachable targets. Phase 2: -/// a real ADS handshake — AdsClient.Connect(netId, port) + ReadStateAsync — to -/// confirm the endpoint speaks ADS and report the controller's run-state, not just that a TCP -/// socket opened. +/// Test-Connect probe for the -shaped driver config. It takes +/// exactly the path the driver takes (): +/// AdsClient.Connect(netId, port) through the local AMS router (TwinCAT XAR, or +/// Beckhoff.TwinCAT.Ads.TcpRouter hosted in-process), followed by one read-only +/// ReadStateAsync, all inside the caller's time budget. /// -/// Outcome classification — three cases: -/// -/// -/// ADS connected + ReadState OK → Ok=true, message "ADS state: {AdsState}" -/// (e.g. "Run" / "Config" / "Stop"), with latency. -/// -/// -/// Route/auth rejection from a reachable router — an -/// (or a non-success ReadStateAsync result) whose means the -/// router answered but won't let us in (e.g. , -/// , , -/// ) → Ok=false, message -/// "Reachable at {host}:{port} but ADS handshake failed: {code} — check the target's ADS -/// route table authorizes this host". This is a TRUE red: the driver itself also needs -/// the route, so a green tick here would be a false positive. -/// -/// -/// Handshake could not be ATTEMPTED on this host — the managed AMS router cannot run -/// headless (Beckhoff's AdsClient.Connect throws a server exception -/// "Check for a running TwinCAT router instance!"), or a -/// / / / -/// surfaces, or ReadStateAsync reports a client-side -/// port-not-open status → DEGRADE: Ok=true, message -/// "Reachable at {host}:{port} (ADS handshake unavailable on this host — TCP reachability -/// only)", with latency. The probe NEVER produces a result worse than the old TCP-only -/// probe. -/// +/// No TCP pre-check (#863). The driver has no IP field: an AMS Net ID is a Beckhoff +/// identifier that the router maps to a host through its route table, and the AMS port +/// (801/851/…) is not a TCP port. The probe therefore never opens a socket of its own, and never +/// dials anything derived from the AMS Net ID. The former first-phase check dialled the first four +/// Net ID octets as an IPv4 address on the AMS port — a host unrelated to the target (often a +/// public internet address) on a port ADS does not use — so Test Connect timed out against every +/// real runtime. +/// +/// +/// Outcome classification — Ok=true only when the target answered: +/// +/// ReadState OK → Ok=true, "ADS state: {AdsState}", with latency. +/// No local AMS router — AdsClient.Connect cannot reach a router on this +/// host (Beckhoff's AdsServerException "Check for a running TwinCAT router +/// instance!"), the client port never opened, or the ADS client cannot load here → +/// Ok=false, "No local AMS router on this host …". The target was not contacted. +/// No route / target unreachable — the router answered but has no route to the Net +/// ID, the target refused the route or port, or the router's own connection to the target +/// failed → Ok=false, "No route to … or target unreachable: {code} …". +/// Timeout — a sync/device timeout code, or the caller's budget expired → +/// Ok=false, "Probe timed out after {n}s …". +/// Any other ADS error → Ok=false, "ADS handshake with … failed: {code}". /// /// /// /// -/// The line between case 2 (device-rejected → RED) and case 3 (can't-attempt → DEGRADE) is the -/// crux. Classification rests on the exception's identity: only an -/// (or a result ) carrying a route/target-port code is a RED — that is the -/// ADS router answering and refusing the route. Beckhoff's TwinCAT.Ads.Server.AdsServerException -/// ("running TwinCAT router instance!") derives from plain , NOT from -/// , so it is correctly classified as "can't attempt → DEGRADE". -/// When genuinely ambiguous, the probe DEGRADES (Ok=true, TCP-only note) rather than emit a false RED. -/// -/// AMS Net ID format is six dot-separated octets (e.g. 192.168.1.10.1.1); the first four -/// are typically the host IPv4 address by Beckhoff convention. The AMS router resolves the real IP -/// route server-side; the probe uses the first-four-octet heuristic for the TCP preflight target, -/// which is reliable for the overwhelming majority of production deployments. -/// -/// The probe is read-only — ReadStateAsync never mutates PLC state — and always disposes the -/// . +/// The probe is read-only — ReadStateAsync never mutates PLC state, and the probe sends no +/// AddRoute — and always disposes the . /// public sealed class TwinCATDriverProbe : IDriverProbe { @@ -71,22 +51,57 @@ public sealed class TwinCATDriverProbe : IDriverProbe Converters = { new JsonStringEnumConverter() }, }; - /// - /// AMS error codes that mean "the router answered but refused the route / target port" — a - /// genuine RED. The driver itself would also be denied, so a green tick would be a false - /// positive. Everything NOT in this set (client-side port/connection errors, sync timeouts, - /// router-not-initialised, etc.) is treated as "couldn't attempt the handshake" → DEGRADE. - /// - private static readonly HashSet _routeRejectCodes = + /// Codes meaning the router on this host is absent, inactive or never gave us a port. + private static readonly HashSet _noRouterCodes = + [ + AdsErrorCode.ClientPortNotOpen, + AdsErrorCode.ClientNoAmsAddr, + AdsErrorCode.RouterNotInitialized, + AdsErrorCode.RouterNotActive, + ]; + + /// + /// Codes meaning the router answered but cannot get us to the target: no route, the route or + /// port was refused, or the router's connection to the target host failed. + /// + private static readonly HashSet _noRouteCodes = [ - AdsErrorCode.TargetPortNotFound, AdsErrorCode.TargetMachineNotFound, + AdsErrorCode.TargetPortNotFound, AdsErrorCode.PortNotConnected, AdsErrorCode.PortDisabled, AdsErrorCode.AccessDenied, AdsErrorCode.DeviceAccessDenied, + AdsErrorCode.HostUnreachable, + AdsErrorCode.WSA_NetworkUnreachable, + AdsErrorCode.WSA_HostDown, + AdsErrorCode.WSA_HostUnreachable, + AdsErrorCode.WSA_HostNotFound, ]; + /// Codes meaning a route exists but the target did not answer in time. + private static readonly HashSet _timeoutCodes = + [ + AdsErrorCode.ClientSyncTimeOut, + AdsErrorCode.SyncTimeOut, + AdsErrorCode.DeviceTimeOut, + AdsErrorCode.WSA_TimedOut, + AdsErrorCode.RouterFragmentTimeout, + ]; + + /// + /// The ADS handshake: connect through the local AMS router and read the target's state. + /// Throws for a non-success ReadState result. Swappable for tests. + /// + private readonly Func> _readState; + + /// Initializes the probe with the real ADS handshake through the local AMS router. + public TwinCATDriverProbe() : this(ReadStateViaLocalRouterAsync) { } + + /// Test seam — substitutes the ADS handshake. + internal TwinCATDriverProbe(Func> readState) + => _readState = readState; + /// public string DriverType => "TwinCAT"; @@ -98,110 +113,90 @@ public sealed class TwinCATDriverProbe : IDriverProbe catch (Exception ex) { return new(false, $"Config JSON is invalid: {ex.Message}", null); } if (opts is null) return new(false, "Config JSON deserialized to null.", null); - var (host, port, parsed) = ExtractTarget(opts); - if (parsed is null || string.IsNullOrWhiteSpace(host) || port <= 0) - return new(false, "Config has no host/port to probe.", null); + var target = TwinCATAmsAddress.TryParse(opts.Devices.FirstOrDefault()?.HostAddress); + if (target is null) + return new(false, "Config has no host/port to probe (expected a device hostAddress ads://{netId}:{port}).", null); - // Phase 1: bare TCP preflight — fast rejection for unreachable hosts. Messages here are - // UNCHANGED from the original TCP-only probe. + var label = $"{target.NetId}:{target.Port}"; + var timedOut = $"Probe timed out after {timeout.TotalSeconds:F0}s waiting for ADS state from {label}"; var sw = Stopwatch.StartNew(); try { - using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - await socket.ConnectAsync(host, port, ct); - } - catch (SocketException ex) - { - return new(false, $"Connect failed: {ex.SocketErrorCode}", null); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - return new(false, $"Probe timed out after {timeout.TotalSeconds:F0}s.", null); - } - catch (Exception ex) - { - return new(false, ex.Message, null); - } - - // Phase 2: real ADS handshake. Connect + ReadStateAsync. The crux is the three-way - // classification of how this can fail — see the class-doc and ClassifyHandshakeFailure. - var degradeNote = $"Reachable at {host}:{port} (ADS handshake unavailable on this host — TCP reachability only)"; - try - { - using var client = new AdsClient(); - // Bound the ADS round-trip by the caller's timeout (clamped >=1s, mirrors AdsTwinCATClient). - client.Timeout = (int)Math.Max(1_000, timeout.TotalMilliseconds); - - // Connect can throw a server exception ("running TwinCAT router instance!") on a headless - // host with no AMS router — that is a can't-attempt DEGRADE, classified below. - client.Connect(AmsNetId.Parse(parsed.NetId), parsed.Port); - - var state = await client.ReadStateAsync(ct).ConfigureAwait(false); + // Task.Run + WaitAsync keep the whole handshake — including the synchronous + // AdsClient.Connect — inside the caller's budget. + var state = await Task.Run(() => _readState(target, timeout, ct), ct) + .WaitAsync(timeout, ct).ConfigureAwait(false); sw.Stop(); - - if (state.Succeeded) - return new(true, $"ADS state: {state.State.AdsState}", sw.Elapsed); - - // Non-throwing failure carried in the result's error code. - return state.ErrorCode == AdsErrorCode.ClientPortNotOpen - ? new(true, degradeNote, sw.Elapsed) // client never opened — DEGRADE - : ClassifyHandshakeFailure(state.ErrorCode, host, port, sw, degradeNote); + return new(true, $"ADS state: {state}", sw.Elapsed); + } + catch (TimeoutException) + { + return new(false, $"{timedOut} — the target did not answer.", null); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - // Caller timeout — keep the original timed-out message. - return new(false, $"Probe timed out after {timeout.TotalSeconds:F0}s.", null); + return new(false, $"{timedOut}.", null); } catch (AdsErrorException ex) { - // The router answered with an ADS-level error. Route/auth rejection → RED; anything - // else (sync timeout, client port issues, …) → DEGRADE. - return ClassifyHandshakeFailure(ex.ErrorCode, host, port, sw, degradeNote); + return Classify(ex.ErrorCode, label, timedOut); } - catch (Exception) + catch (Exception ex) when (IsNoRouter(ex)) { - // Everything else — TwinCAT.Ads.Server.AdsServerException ("running TwinCAT router - // instance!"), PlatformNotSupportedException, TypeInitializationException, - // DllNotFoundException, NotSupportedException, etc. — means the handshake could not be - // ATTEMPTED on this host. DEGRADE: never worse than the TCP-only probe. - sw.Stop(); - return new(true, degradeNote, sw.Elapsed); + return new(false, NoRouterMessage(ex.Message), null); + } + catch (Exception ex) + { + return new(false, $"ADS handshake with {label} failed: {ex.Message}", null); } } + /// Default handshake — the same calls makes. + private static async Task ReadStateViaLocalRouterAsync( + TwinCATAmsAddress target, TimeSpan timeout, CancellationToken ct) + { + using var client = new AdsClient(); + client.Timeout = (int)Math.Max(1_000, timeout.TotalMilliseconds); + client.Connect(AmsNetId.Parse(target.NetId), target.Port); + + var state = await client.ReadStateAsync(ct).ConfigureAwait(false); + if (!state.Succeeded) + throw new AdsErrorException($"ReadState failed: {state.ErrorCode}", state.ErrorCode); + return state.State.AdsState; + } + + /// Maps an ADS error code to one of the probe's failure classes. + internal static DriverProbeResult Classify(AdsErrorCode code, string label, string timedOut) + { + if (_noRouterCodes.Contains(code)) + return new(false, NoRouterMessage(code.ToString()), null); + if (_noRouteCodes.Contains(code)) + return new(false, + $"No route to {label} or target unreachable: {code} — check the local AMS router has a route to this AMS Net ID and the target's route table authorizes this host.", + null); + if (_timeoutCodes.Contains(code)) + return new(false, $"{timedOut} ({code}) — the router has a route but the target did not answer.", null); + return new(false, $"ADS handshake with {label} failed: {code}", null); + } + + private static string NoRouterMessage(string detail) => + $"No local AMS router on this host — start TwinCAT XAR or host the Beckhoff TcpRouter; the target was not contacted ({detail})."; + /// - /// Classifies an ADS-level failure (from an or a non-success - /// ReadStateAsync result). A route/target-port/access code means the router answered - /// but refused the route → RED. Any other code is treated as "couldn't attempt" → DEGRADE, - /// so the probe never under-reports a host with no usable ADS runtime. + /// True when the ADS client could not reach a router on this host, or cannot load here at all. + /// Beckhoff's AdsServerException / AmsServerException ("Check for a running TwinCAT + /// router instance!") derive from plain , so they are matched by type name. /// - private static DriverProbeResult ClassifyHandshakeFailure( - AdsErrorCode code, string host, int port, Stopwatch sw, string degradeNote) + private static bool IsNoRouter(Exception ex) { - if (_routeRejectCodes.Contains(code)) - return new(false, - $"Reachable at {host}:{port} but ADS handshake failed: {code} — check the target's ADS route table authorizes this host", - null); - - sw.Stop(); - return new(true, degradeNote, sw.Elapsed); - } - - private static (string host, int port, TwinCATAmsAddress? parsed) ExtractTarget(TwinCATDriverOptions opts) - { - // Parse the first device's ads:// address. AMS Net ID is six-octet; by Beckhoff - // convention the first four octets are the host IPv4. Extract those as the TCP target. - var firstDevice = opts.Devices.FirstOrDefault(); - if (firstDevice is null) return (string.Empty, 0, null); - - var parsed = TwinCATAmsAddress.TryParse(firstDevice.HostAddress); - if (parsed is null) return (string.Empty, 0, null); - - // NetId = "a.b.c.d.e.f" — take the first 4 octets as the host IP. - var parts = parsed.NetId.Split('.'); - if (parts.Length < 4) return (string.Empty, 0, null); - - var hostIp = string.Join('.', parts[0], parts[1], parts[2], parts[3]); - return (hostIp, parsed.Port, parsed); + for (var e = ex; e is not null; e = e.InnerException) + { + if (e is PlatformNotSupportedException or TypeInitializationException or DllNotFoundException) + return true; + var name = e.GetType().FullName; + if (name is "TwinCAT.Ads.Server.AdsServerException" or "TwinCAT.Ams.AmsServerException") + return true; + } + return false; } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATDriverProbeTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATDriverProbeTests.cs index ec0ca8628..01bdf68ae 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATDriverProbeTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATDriverProbeTests.cs @@ -1,22 +1,17 @@ -// Unit tests for TwinCATDriverProbe — the degrade-guarded ADS ReadState Test-Connect probe. +// Unit tests for TwinCATDriverProbe — the ADS ReadState Test-Connect probe (#863). // -// Coverage here is the set of outcomes that are *deterministic on a CI/dev box with no TwinCAT -// runtime*: invalid JSON, missing host/port, an unreachable TCP target, and — the crux — the -// DEGRADE path. On this box there is no AMS router, so once the TCP preflight succeeds against a -// reachable listener the managed ADS client cannot initialise (it throws a "Check for a running -// TwinCAT router instance!" server exception, NOT an ADS device rejection). The probe MUST treat -// that as "TCP reachability only" (Ok=true) so it never regresses below the old TCP-only probe. -// -// The two paths that REQUIRE a live ADS target are LIVE-VERIFY DEFERRED (no TwinCAT runtime on the -// rig): (a) the happy "ADS state: Run/Config/Stop" path (Ok=true with the AdsState name), and -// (b) the route/auth-reject RED path (a reachable router that refuses the route, surfaced as an -// AdsErrorException carrying a TargetPortNotFound / route AdsErrorCode → Ok=false with the -// "check the target's ADS route table" message). Those are exercised against a real TwinCAT XAR / -// remote runtime, out of scope for this unit suite. +// The probe takes exactly the driver's path: AdsClient.Connect(netId, port) through the LOCAL AMS +// router, then one ReadStateAsync. It must never open a socket of its own, and never to a host derived +// from the AMS Net ID (the old first-phase check dialled the Net ID's first four octets on the AMS +// port — an unrelated host, often a public one, on a port ADS does not use — so Test Connect timed out +// against every real runtime). The classification and budget tests drive the handshake through the +// internal seam; the no-dial and no-router tests use the real AdsClient on a box with no AMS router. +using System.Diagnostics; using System.Net; using System.Net.Sockets; using Shouldly; +using TwinCAT.Ads; using Xunit; namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; @@ -24,199 +19,162 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; [Trait("Category", "Unit")] public sealed class TwinCATDriverProbeTests { - private static readonly TwinCATDriverProbe Probe = new(); + private const string Z35474Like = "{\"devices\":[{\"hostAddress\":\"ads://5.118.56.93.1.1:851\"}]}"; - // ------------------------------------------------------------------------- - // 1. Invalid JSON - // ------------------------------------------------------------------------- + private static TwinCATDriverProbe ProbeWith(Func> handshake) + => new(handshake); + + // ---- config validation ---------------------------------------------------------------- - /// Invalid JSON returns Ok=false with a message containing "invalid". [Fact] public async Task InvalidJson_Returns_OkFalse_WithInvalidMessage() { - var result = await Probe.ProbeAsync( - "not-valid-json{{{", - TimeSpan.FromSeconds(3), - TestContext.Current.CancellationToken); + var result = await new TwinCATDriverProbe().ProbeAsync( + "not-valid-json{{{", TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); result.Ok.ShouldBeFalse(); - result.Message.ShouldNotBeNull(); result.Message!.ToLowerInvariant().ShouldContain("invalid"); result.Latency.ShouldBeNull(); } - // ------------------------------------------------------------------------- - // 2. Config with no device / no host - // ------------------------------------------------------------------------- - - /// An empty Devices list returns Ok=false with a "no host/port" message. - [Fact] - public async Task NoDevices_Returns_OkFalse_WithNoHostPortMessage() + [Theory] + [InlineData("{\"devices\":[]}")] + [InlineData("{\"devices\":[{\"hostAddress\":\"not-an-ads-url\"}]}")] + public async Task NoUsableDevice_Returns_OkFalse_WithNoHostPortMessage(string json) { - var result = await Probe.ProbeAsync( - "{\"devices\":[]}", - TimeSpan.FromSeconds(3), - TestContext.Current.CancellationToken); + var called = false; + var result = await ProbeWith((_, _, _) => { called = true; return Task.FromResult(AdsState.Run); }) + .ProbeAsync(json, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); result.Ok.ShouldBeFalse(); - result.Message.ShouldNotBeNull(); result.Message!.ShouldContain("no host/port"); - result.Latency.ShouldBeNull(); + called.ShouldBeFalse(); } - /// - /// A first device with a malformed (non-ads://) host address returns Ok=false with a - /// "no host/port" message because returns null. - /// - [Fact] - public async Task MalformedHostAddress_Returns_OkFalse_WithNoHostPortMessage() - { - var result = await Probe.ProbeAsync( - "{\"devices\":[{\"hostAddress\":\"not-an-ads-url\"}]}", - TimeSpan.FromSeconds(3), - TestContext.Current.CancellationToken); - - result.Ok.ShouldBeFalse(); - result.Message.ShouldNotBeNull(); - result.Message!.ShouldContain("no host/port"); - result.Latency.ShouldBeNull(); - } - - // ------------------------------------------------------------------------- - // 3. Unreachable target — TCP preflight fails first - // ------------------------------------------------------------------------- + // ---- #863: never dial a host derived from the AMS Net ID -------------------------------- /// - /// A closed port causes the TCP preflight ConnectAsync to throw - /// ; the probe returns Ok=false with "Connect failed". - /// The port is reserved then released so the OS refuses the connection immediately. + /// Regression for #863. A listener sits on exactly the endpoint the old probe dialled + /// (Net ID octets 1–4 as IPv4, AMS port as TCP port). The real probe must never connect to + /// it — the old TCP pre-check connected here first, so this fails on the old behaviour. /// [Fact] - public async Task ClosedPort_Returns_OkFalse_WithConnectFailedMessage() + public async Task Probe_NeverConnects_ToAmsNetIdDerivedEndpoint() { - // Reserve an ephemeral port then release it so the port is definitely closed. - var reserved = new TcpListener(IPAddress.Loopback, 0); - reserved.Start(); - var port = ((IPEndPoint)reserved.LocalEndpoint).Port; - reserved.Stop(); - - // 127.0.0.1 as the first four AMS-Net-ID octets → the probe's TCP target is 127.0.0.1. - var configJson = $"{{\"devices\":[{{\"hostAddress\":\"ads://127.0.0.1.1.1:{port}\"}}]}}"; - - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - TestContext.Current.CancellationToken); - cts.CancelAfter(TimeSpan.FromSeconds(5)); - - var result = await Probe.ProbeAsync( - configJson, - TimeSpan.FromSeconds(3), - cts.Token); - - result.Ok.ShouldBeFalse(); - result.Message.ShouldNotBeNull(); - // Either a fast SocketException ("Connect failed") or, if the OS stalls, a timeout. - var lower = result.Message!.ToLowerInvariant(); - (lower.Contains("connect failed") || lower.Contains("timed out")).ShouldBeTrue( - $"Expected 'Connect failed' or 'timed out' but got: {result.Message}"); - result.Latency.ShouldBeNull(); - } - - /// - /// A target on a non-routable IP (192.0.2.1 — TEST-NET-1, RFC 5737) with a short - /// cancellation window times out (or fails fast on EHOSTUNREACH) at the TCP preflight. - /// - [Fact] - public async Task BlackHoleTarget_Returns_OkFalse_WithTimedOutOrConnectFailed() - { - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - TestContext.Current.CancellationToken); - cts.CancelAfter(TimeSpan.FromSeconds(1)); - - var configJson = "{\"devices\":[{\"hostAddress\":\"ads://192.0.2.1.1.1:851\"}]}"; - var result = await Probe.ProbeAsync( - configJson, - TimeSpan.FromSeconds(1), - cts.Token); - - result.Ok.ShouldBeFalse(); - result.Message.ShouldNotBeNull(); - var lower = result.Message!.ToLowerInvariant(); - (lower.Contains("timed out") || lower.Contains("connect failed")).ShouldBeTrue( - $"Expected 'timed out' or 'Connect failed' but got: {result.Message}"); - result.Latency.ShouldBeNull(); - } - - // ------------------------------------------------------------------------- - // 4. DEGRADE path — TCP reachable, but the ADS handshake can't be attempted here - // ------------------------------------------------------------------------- - - /// - /// The crux. A reachable TCP listener satisfies the preflight, but on this box (no TwinCAT - /// AMS router) the managed ADS client cannot establish a session — AdsClient.Connect - /// throws an environment/router exception (NOT an ADS device rejection). The probe MUST - /// DEGRADE: Ok=true with the "TCP reachability only" note, never a false RED. This - /// guarantees the new probe never reports worse than the old TCP-only probe on a host with - /// no ADS runtime. - /// - [Fact] - public async Task ReachableTcp_NoAdsRuntime_Degrades_To_OkTrue_WithTcpOnlyNote() - { - // Stand up a real TCP listener and accept (then drop) connections so the preflight - // ConnectAsync succeeds. We never speak ADS — the ADS handshake is what must degrade. var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; - - using var acceptCts = CancellationTokenSource.CreateLinkedTokenSource( - TestContext.Current.CancellationToken); - var acceptLoop = Task.Run(async () => - { - try - { - while (true) - { - var client = await listener.AcceptTcpClientAsync(acceptCts.Token); - // Hold the socket briefly so the preflight sees a clean accept, then close. - _ = Task.Run(async () => - { - await Task.Delay(200); - client.Dispose(); - }); - } - } - catch (OperationCanceledException) { /* accept cancelled — expected on teardown */ } - catch (ObjectDisposedException) { /* listener stopped — expected on teardown */ } - catch (SocketException) { /* listener stopped — expected on teardown */ } - }, TestContext.Current.CancellationToken); - try { - var configJson = $"{{\"devices\":[{{\"hostAddress\":\"ads://127.0.0.1.1.1:{port}\"}}]}}"; - - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - TestContext.Current.CancellationToken); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); + await new TwinCATDriverProbe().ProbeAsync( + $"{{\"devices\":[{{\"hostAddress\":\"ads://127.0.0.1.1.1:{port}\"}}]}}", + TimeSpan.FromSeconds(3), cts.Token); - var result = await Probe.ProbeAsync( - configJson, - TimeSpan.FromSeconds(3), - cts.Token); - - // DEGRADE guard: must be Ok=true (never worse than TCP-only), with the explicit note - // and a latency. If on some box AdsClient instead threw synchronously at construction, - // the probe still degrades the same way — this assertion holds either way. - result.Ok.ShouldBeTrue( - $"Degrade guard violated — probe returned a worse-than-TCP result: {result.Message}"); - result.Message.ShouldNotBeNull(); - result.Message!.ShouldContain("ADS handshake unavailable on this host"); - result.Message!.ShouldContain("TCP reachability only"); - result.Latency.ShouldNotBeNull(); + listener.Pending().ShouldBeFalse( + "the probe opened a TCP connection to the AmsNetId-derived address — it must only use the local AMS router"); } finally { - await acceptCts.CancelAsync(); listener.Stop(); - try { await acceptLoop; } catch { /* ignore teardown races */ } } } + + /// The handshake receives the AMS address unchanged — the probe resolves no host itself. + [Fact] + public async Task Handshake_IsHandedTheAmsAddress_Unchanged() + { + TwinCATAmsAddress? seen = null; + var result = await ProbeWith((t, _, _) => { seen = t; return Task.FromResult(AdsState.Run); }) + .ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + result.Ok.ShouldBeTrue(); + seen.ShouldBe(new TwinCATAmsAddress("5.118.56.93.1.1", 851)); + } + + // ---- outcome classification ------------------------------------------------------------- + + [Fact] + public async Task ReadStateOk_Returns_OkTrue_WithStateAndLatency() + { + var result = await ProbeWith((_, _, _) => Task.FromResult(AdsState.Run)) + .ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + result.Ok.ShouldBeTrue(); + result.Message.ShouldBe("ADS state: Run"); + result.Latency.ShouldNotBeNull(); + } + + /// + /// On a box with no AMS router the real AdsClient cannot connect. That is a RED naming the + /// missing router — no longer a green "TCP reachability only", since nothing was contacted. + /// + [Fact] + public async Task NoLocalRouter_RealClient_Returns_OkFalse_NamingTheRouter() + { + var result = await new TwinCATDriverProbe().ProbeAsync( + Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + result.Ok.ShouldBeFalse(); + result.Message!.ShouldContain("No local AMS router"); + result.Message!.ShouldContain("not contacted"); + result.Latency.ShouldBeNull(); + } + + [Fact] + public async Task AmsServerException_Classified_AsNoLocalRouter() + { + var result = await ProbeWith((_, _, _) => throw new global::TwinCAT.Ams.AmsServerException("Connect to Server failed!")) + .ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + result.Ok.ShouldBeFalse(); + result.Message!.ShouldContain("No local AMS router"); + } + + [Theory] + [InlineData(AdsErrorCode.ClientPortNotOpen, "No local AMS router")] + [InlineData(AdsErrorCode.TargetMachineNotFound, "No route to 5.118.56.93.1.1:851 or target unreachable")] + [InlineData(AdsErrorCode.TargetPortNotFound, "No route to 5.118.56.93.1.1:851 or target unreachable")] + [InlineData(AdsErrorCode.WSA_HostUnreachable, "No route to 5.118.56.93.1.1:851 or target unreachable")] + [InlineData(AdsErrorCode.ClientSyncTimeOut, "timed out")] + [InlineData(AdsErrorCode.DeviceInvalidGroup, "ADS handshake with 5.118.56.93.1.1:851 failed")] + public async Task AdsErrorCodes_AreClassified(AdsErrorCode code, string expected) + { + var result = await ProbeWith((_, _, _) => throw new AdsErrorException("x", code)) + .ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + result.Ok.ShouldBeFalse(); + result.Message!.ShouldContain(expected); + result.Message!.ShouldContain(code.ToString()); + result.Latency.ShouldBeNull(); + } + + // ---- time budget -------------------------------------------------------------------------- + + /// A handshake that never returns (and ignores its token) is cut off at the budget. + [Fact] + public async Task HangingHandshake_IsBoundedByTimeout() + { + var never = new TaskCompletionSource(); + var sw = Stopwatch.StartNew(); + var result = await ProbeWith((_, _, _) => never.Task) + .ProbeAsync(Z35474Like, TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + sw.Stop(); + + result.Ok.ShouldBeFalse(); + result.Message!.ShouldContain("timed out after 1s"); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task CallerCancellation_Returns_TimedOut() + { + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + var result = await ProbeWith(async (_, _, ct) => { await Task.Delay(Timeout.Infinite, ct); return AdsState.Run; }) + .ProbeAsync(Z35474Like, TimeSpan.FromSeconds(10), cts.Token); + + result.Ok.ShouldBeFalse(); + result.Message!.ShouldContain("Probe timed out after 10s"); + } } From d9b23e0b958673c67876db8ec5fdc15970c5d990 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:18:11 -0500 Subject: [PATCH 07/28] feat(opcua): per-item delivery for folder event items; folder ConditionRefresh gets the #854 cut (#855) Condition events reaching one of our notifiers no longer go through the SDK MonitoredNode2 channel: the walk serves the attached items itself, per item and per event (audit guard, uncached INodeManager3 verdict, item gate/cut, QueueEvent), as the Server-object path does. No notifier of ours defers any more; the #838 deferral queue and the #854 cut stay as the safety net and folder items share the cut (RefreshDirectItems, flag on and off). An event whose walk suspends is marked deferred before it suspends. Tests produce deferral with a notifier-stall probe instead of holding a folder item's SDK lock (which would now block the reporter). New FolderConditionRefreshWireTests; design note in docs/plans. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .claude/rules/data-plane-auth.md | 2 +- ...26-11-11-folder-event-per-item-delivery.md | 102 +++ docs/security.md | 63 +- .../OtOpcUaNodeManager.cs | 520 ++++++++++++--- .../ConditionRefreshDeferralWireTests.cs | 70 +- .../EventQueueOverflowAclWireTests.cs | 31 +- .../FolderConditionRefreshWireTests.cs | 627 ++++++++++++++++++ .../FolderEventFloodWireTests.cs | 50 +- .../FolderEventVerdictInvalidationTests.cs | 8 +- .../NotifierStall.cs | 42 ++ 10 files changed, 1302 insertions(+), 213 deletions(-) create mode 100644 docs/plans/2026-11-11-folder-event-per-item-delivery.md create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs diff --git a/.claude/rules/data-plane-auth.md b/.claude/rules/data-plane-auth.md index a93cb46bd..ded979d62 100644 --- a/.claude/rules/data-plane-auth.md +++ b/.claude/rules/data-plane-auth.md @@ -217,7 +217,7 @@ Two things to know before relying on any of this: replay off `Lock` one event per hold of a per-item gate, and withholds older deferred events from THAT item only (re-sent condition, or Retain=true of one omitted because NOT retained — never one omitted by ACL). Item state is a `ConditionalWeakTable` keyed by the item; cuts clear when the deferral count hits 0. Tests: - `ConditionRefreshDeferralWireTests`; folder items keep the residual (SDK channel + memoised verdict). Folder events follow Part 3's shape — + `ConditionRefreshDeferralWireTests`. ⚠️ **Since #855 folder items are delivered per item too:** at every notifier we own, `DeliverConditionEventAsync` serves the `MonitoredNode2`'s attached items itself (`DeliverToNotifierItems`: audit guard → `INodeManager3.ValidateEventRolePermissions` per event, UNCACHED → item gate/cut → `QueueEvent`), never through the SDK channel — so no notifier of ours defers, a flood under `Lock` defers nothing, and folder refreshes (`RefreshDirectItems`, flag on AND off) share the #854 cut. The deferral queue + cut remain as the safety net; an event whose walk suspends is marked deferred BEFORE suspending. Tests produce deferral with `NotifierStall` (the `NotifierDeliveryStall` probe) — holding a folder item's SDK lock now blocks the reporter. `FolderConditionRefreshWireTests`; `docs/plans/2026-11-11-folder-event-per-item-delivery.md`. Folder events follow Part 3's shape — `ReceiveEvents` on EventType + SourceNode (SDK check, kept first) AND our decision; non-condition events on a folder item fail closed under enforcement. Tests: `FolderEventFloodWireTests`. Overflow WITH the ACL doing work (#838 item 2) — denied condition interleaved, folder + `i=2253` items, non-publishing client, concurrent Read, grant revoked diff --git a/docs/plans/2026-11-11-folder-event-per-item-delivery.md b/docs/plans/2026-11-11-folder-event-per-item-delivery.md new file mode 100644 index 000000000..72ea22512 --- /dev/null +++ b/docs/plans/2026-11-11-folder-event-per-item-delivery.md @@ -0,0 +1,102 @@ +# Folder (notifier) event items: per-item delivery (Gitea #855) + +Status: design for #855 (owner decision 2026-11-11: build per-item delivery). Background: #838 (deferral instead of +blocking under `Lock`), #854 (sequence cut on Server-object refresh), #826 (per-folder event ACL). + +## Problem + +A condition event reaches an event item on an equipment / raw device folder through the SDK's `MonitoredNode2` +for that folder: `OnReportEventAsync` writes it to a bounded channel (4096), and a per-node consumer task decides +each item's verdict (memoised per `(item, event type, SourceNode)`) and calls `QueueEvent`. That gives the node +manager no per-event, per-item decision point, so a `ConditionRefresh` of a folder item cannot withhold an older +event that is still in the channel or in our deferral queue: the refresh is overtaken (the #854 guarantee covers only +Server-object items). The channel's consumer also needs `Lock` on a verdict miss, which is why a burst under `Lock` +fills the channel and defers (#838). + +## Design + +### Where delivery hooks in + +`DeliverConditionEventAsync` already walks the condition's notifier chain hop by hop (our copy of the SDK's +`ReportEventAsync`, #838). At a hop whose `OnReportEventAsync` is the SDK `MonitoredNode2` of **a node this manager +owns** (`handler.Target is MonitoredNode2 m && m.Node == node`), the walk no longer calls the handler. It delivers the +event itself to that node's attached items (`m.EventMonitoredItems`), one by one: the SDK's audit guard, the verdict +(`INodeManager3.ValidateEventRolePermissions` — the SDK's type/source `ReceiveEvents` check plus, under enforcement, +the #826 live-notifier + grant check), then the item's gate and cut, then `QueueEvent`. + +This is exactly how Server-object items are served (`OnConditionReportEvent`, #586/#854): same verdict per event, same +gate, same cut, same `QueueEvent`. + +### Bypass / coexistence with `MonitoredNode2` + +- **Condition events bypass the channel** for our notifiers. The `MonitoredNode2` stays: the SDK still owns item + attachment (subscribe / unsubscribe / transfer / tombstone / revalidation stand-down detach), and its channel still + carries everything else that reaches the node — a condition's SDK-raised method **audit** events (only with auditing + on) and data-change notifications for attribute items on the folder. Those keep the SDK's cached verdict, so + `InvalidateFolderEventVerdicts` stays. +- A stood-down item is not in `EventMonitoredItems`, so it receives nothing, as before. +- A hop whose handler is anything else (none exists today: conditions are never event notifiers and variables cannot + host event items) still goes through the handler — the fallback that keeps the #838 deferral reachable. + +### Consequence for overload + +Every hop the node manager owns now completes synchronously: `QueueEvent` never blocks (the item's own queue discards +oldest at `QueueSize`), and the verdict runs on the reporter's thread, which already holds `Lock`. So a burst under one +`Lock` hold no longer defers at all, and the #838 deadlock shape (a consumer needing `Lock` while the reporter waits on +its channel) is gone for condition events. Folder items stay in report order per item, exactly like Server-object items. + +The #838 deferral queue and the #854 cut machinery remain as the safety net for a hop that does suspend (the fallback +above, or a future SDK change). Folder items are brought under the same cut: + +- An event whose delivery continues off `Lock` — queued in the deferral queue, **or** the in-flight event whose walk + suspended at a hop (new: marked when the walk suspends, so the hops after the suspension take the gate) — takes the + item's gate and is withheld when the item's latest refresh cut supersedes it (`RefreshCut.Supersedes`, unchanged). +- `ConditionRefresh` / `ConditionRefresh2` of folder items has the same two paths as the Server object, flag on and off: + - **Nothing deferred:** the SDK base walk of the item's notifier (off `Lock`, live condition nodes), per-item verdict + as before (#826 filter under enforcement). Kept only if no deferral episode existed at any moment of it; otherwise + the cut path runs too and supersedes it inside the same RefreshStart..RefreshEnd. Its closing check takes `Lock` + once: a report whose walk suspended (marked under `Lock` before suspending) has registered its deferral by then — + without it a folder refresh could pass the check in the instant between the suspension and the count. + - **Deferring — the cut:** verdicts per (item, condition) first, off `Lock`, from a live walk. Then one `Lock` hold: + watermark, episode, the Retain set, and per item the same walk producing frozen snapshots, with the cut + "replay + pending" marker installed under the item's gate. The replay is queued off `Lock`, one event per gate hold. +- State per item (`EventItemState`, previously Server-object only) is keyed by the item object for its lifetime, so it + survives unsubscribe + resubscribe; stale cuts are cleared when the deferral episode ends, for folder and Server items. + +### Locking + +Unchanged order, now for folder items too: `Lock` → item gate → the SDK item's own lock. Synchronous delivery runs +under the reporter's `Lock`; the verdict takes `Lock` re-entrantly and the core node manager's / authorizer's locks +(as the Server-object path already does). Off-`Lock` (deferred) delivery computes the verdict **before** taking the item +gate, so nothing under a gate takes `Lock` or the event-report gate. A reporter waits at most for one in-progress +`QueueEvent` on an item; a refresh waits at most for one deferred delivery already inside the gate, never for a drain. +No reporter ever waits on a channel. + +### Bounds + +No new queue. Per item: the SDK item queue (`QueueSize`, DiscardOldest). Deferral (only on the fallback): unchanged, +1024 per condition / 65 536 total, coalescing. Cuts: one per refreshed item, sets sized by the conditions, cleared at the +end of the deferral episode. + +### ACL (#826) + +Each event is decided per item at delivery against the current generation — no memoised verdict for condition events +any more, so a deploy / revocation / reference removal takes effect on the next event without waiting for an +invalidation. Refresh replays only conditions the item may see (the same check); a condition omitted for want of a grant +is never withheld afterwards (the cut's existing rule). + +### Cost + +The per-event verdict for folder items moves from the SDK's consumer task (cached) to the reporter thread under `Lock` +(uncached), as Server-object items already pay. Folder subscribers are few (ScadaBridge subscribes at `i=2253`). + +## Tests + +- `ConditionRefreshDeferralWireTests` / `FolderEventFloodWireTests` / `EventQueueOverflowAclWireTests`: deferral is no + longer produced by flooding a folder item; tests that need it stall the folder's hop through a test probe + (`NotifierDeliveryStall`, null in production) instead of the folder item's SDK lock (which would now block the reporter, + as it would on the Server object). Flood tests now pin that folder delivery never defers. +- New `FolderConditionRefreshWireTests`: folder-item refresh during deferral (flag on/off, retained / cleared, + ConditionRefresh2, resubscribe), the gate race, newer-than-cut replay, ACL omission not withheld, ACL revocation during + deferral, the in-flight suspended event, and two stress tests (production overload with no stall; stall + refresh + + reports concurrently). diff --git a/docs/security.md b/docs/security.md index ec400d57b..1e09644b1 100644 --- a/docs/security.md +++ b/docs/security.md @@ -901,7 +901,7 @@ Behaviours worth knowing before enabling it: older `Retain=false` event of such a condition still arrives (it agrees with the omission), and a condition that was retained but omitted for the item's **grants** is never withheld (a grant added before the drain lets its events through; the ACL decides them at delivery as ever). - - The deferral queues, every other Server-object item and every folder item are untouched. A refresh waits for the + - The deferral queues, every other Server-object item and every non-refreshing folder item are untouched. A refresh waits for the drain at most for one in-progress `QueueEvent` on that item (the gate), never for the drain itself; a reporter holding `Lock` likewise waits at most for one. Each item keeps one gate/cut state for its lifetime (keyed by the item, so an unsubscribe + resubscribe keeps it), and cuts are cleared when the deferral count returns to zero. @@ -912,29 +912,48 @@ Behaviours worth knowing before enabling it: - **Measured** (3000 retained conditions, longest `Lock` wait another thread saw during one refresh): the first #854 cut (`1622d07f`, everything under `Lock`) 95–106 ms idle / 102–127 ms deferring; now 0–3 ms idle / 23–32 ms deferring (the frozen snapshots). - - ⚠️ **Residual: folder (notifier) items.** A refresh of an item on an equipment/device folder can still be followed - by older events of a condition. Two sources, neither reachable per item: the SDK's own `MonitoredNode2` channel - (up to 4096 events already accepted before the refresh — this exists in the stock SDK too, and in normal operation - for the few events in transit), and our deferral queue, whose folder hop goes through that channel, where the SDK - memoises the per-item permission verdict by (item, event type, source) so there is no per-event, per-item hook. The - events stay in the condition's order and its newest arrives last, so the client converges. Discarding the queue - globally on refresh would drop transitions for every non-refreshing subscriber while leaving the channel backlog; - making refresh wait would block a service call on the stalled drain. ScadaBridge also ignores a transition older - than the one it holds for a condition. + - **Folder (notifier) items are delivered per item (Gitea #855).** Before #855 a folder item was fed by the SDK's own + `MonitoredNode2` channel (up to 4096 events accepted before a refresh), with the per-item verdict memoised by (item, + event type, source), so a refresh of a folder item could still be followed by older events. Now, at every notifier + the node manager owns, the walk serves the attached items itself (`DeliverToNotifierItems`), exactly as it serves + Server-object items: the SDK's audit guard, the verdict through `INodeManager3.ValidateEventRolePermissions` (SDK + type/source `ReceiveEvents` + the #826 decision under enforcement) **per event, uncached**, then the item's gate and + cut, then `QueueEvent`. The `MonitoredNode2` stays for item attachment and for what still uses its channel (a + condition's SDK method audit events, data-change items on the folder), so `InvalidateFolderEventVerdicts` stays. + - **No notifier of ours defers any more.** `QueueEvent` never waits for a consumer (the item queue discards oldest at + `QueueSize`) and the verdict runs on the reporter's thread, which already holds `Lock` — so a burst under one + `Lock` hold completes with nothing deferred, and the #838 deadlock shape is gone for condition events. Folder items + therefore see condition events in report order. The cost moves with it: the per-event verdict and filter for + folder items now run under `Lock` (as for Server-object items), where the SDK used to cache the verdict off it. + - **The deferral queue and the cut stay as the safety net**, for a hop that is not ours (none exists today: + conditions are never event notifiers and variables cannot host event items) — and folder items share the cut: + a `ConditionRefresh`/`ConditionRefresh2` of a folder item takes the same two paths as the Server object (plain walk + off `Lock` when nothing is deferred; otherwise verdicts off `Lock`, then one `Lock` hold for the watermark, the + Retain set and, per item, its notifier walk frozen as snapshots, with the cut installed under the item gate and the + replay queued off `Lock`). An event whose walk suspended at a hop is marked deferred **before** it suspends, so + every later hop judges it against the cut; the plain refresh path's closing check takes `Lock` once, so a report + that suspended under it has registered its deferral by then. Lock order is unchanged (`Lock` → item gate → item + lock); off `Lock` the verdict is computed before the gate. Pinned by `FolderEventFloodWireTests`: 6000 events under one `Lock` hold with publishing disabled (flag on and off) - complete, drain to zero, coalesce, and the item delivers at most its revised `QueueSize` in increasing order ending - with the last event; and a condition **removed** in the same `Lock` hold has none of its deferred events delivered to - either the folder or the Server-object item (the flag-off variant fails without the liveness checks). - `ConditionRefreshDeferralWireTests` (#854) stalls a folder item's SDK lock so its channel fills without `Lock` held, - refreshes the Server-object item (ConditionRefresh and ConditionRefresh2, flag on/off, condition retained or cleared) - and pins that nothing older than the replay follows RefreshEnd while the non-refreshing folder item still receives the - whole tail (also across an unsubscribe + resubscribe of the item, and the cut is cleared after the drain); a - probe-driven test pins that a refresh waits for a delivery already past its check (fails without the item gate); an - ACL-on test pins that a condition omitted for want of a grant is not withheld once the grant lands; a 3000-condition - test measures the `Lock` hold and pins that neither path queues its replay under `Lock`; and a stress test runs - reports, refreshes and an intermittent stall concurrently (its stream never goes backwards outside a refresh, and - every refresh ends on the newest state delivered). + complete **with nothing deferred** (fails if the per-item delivery stops recognising the SDK fan-out), and the item + delivers at most its revised `QueueSize` in increasing order ending with the last event; and a condition **removed** + while its events wait (delivery stalled at the folder) has none of them delivered to either the folder or the + Server-object item. Tests that need a deferral stall delivery at a folder through a test probe + (`NotifierStall` → `OtOpcUaNodeManager.NotifierDeliveryStall`, null in production), which suspends the walk exactly + as a full channel did — holding a folder item's SDK lock, as the pre-#855 tests did, would now block the reporter. + `ConditionRefreshDeferralWireTests` (#854) refreshes the Server-object item while the folder is stalled + (ConditionRefresh and ConditionRefresh2, flag on/off, condition retained or cleared) and pins that nothing older than + the replay follows RefreshEnd while the non-refreshing folder item still receives the whole tail (also across an + unsubscribe + resubscribe of the item, and the cut is cleared after the drain); a probe-driven test pins that a + refresh waits for a delivery already past its check; an ACL-on test pins that a condition omitted for want of a grant + is not withheld once the grant lands; a 3000-condition test measures the `Lock` hold; and a stress test runs reports, + refreshes and an intermittent stall concurrently. `FolderConditionRefreshWireTests` (#855) is the folder-item twin: + the same matrix on a refreshed folder item (a second subscriber on the same folder keeps the whole tail; a + ModifyMonitoredItems re-subscribe keeps the cut), the gate race, a newer-than-cut event on the raw device folder, a + condition omitted by the ACL (reference dropped) not withheld once it returns, a revocation after the refresh, a + production-shape overload (bursts under `Lock` + repeated refreshes, no stall: nothing deferred, the stream never + goes backwards, no replay queued under `Lock`) and a stress test with an intermittent stall. **Model-change and audit events on the Server object are gated too** (Gitea #588). The node manager's own server-wide events — `GeneralModelChangeEvent` (NodeAdded / NodeDeleted / DataTypeChanged) and the diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index 050202a84..c3d96cc82 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -1476,7 +1476,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// synchronous: the walk completes inline while every channel has room. /// /// - /// Deferral. When a channel is full the walk continues asynchronously (holding no thread or lock), + /// Since Gitea #855 no notifier of ours is a channel. At a notifier this manager owns, the walk serves + /// the items itself () instead of writing to the SDK's + /// MonitoredNode2 channel, so it always completes inline and the deferral below is reached only through + /// a hop that is not ours (none exists today) — or the test stall (). It is + /// kept as the safety net, and the refresh cut (#854) covers folder items too. + /// + /// + /// Deferral. When a hop suspends the walk continues asynchronously (holding no thread or lock), /// and the condition gets a queue: its later events wait behind it, so ONE condition's events stay in /// order, while other conditions whose channels have room are not held up (per-condition, not global). /// Events of different conditions may interleave differently than reported — each carries its own @@ -1615,7 +1622,30 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 node.OnReportEvent?.Invoke(SystemContext, node, e); if (node.OnReportEventAsync is { } handler) - await handler(SystemContext, node, e, CancellationToken.None).ConfigureAwait(false); + { + if (OwnNotifierFanOut(node, handler) is { } monitored) + { + // Gitea #855: one of our notifiers. Its items are served here, per item and per event, exactly as the + // Server-object items are (OnConditionReportEvent) — never through the SDK's channel, whose consumer + // needed Lock (#838) and gave no per-event, per-item point at which a refresh cut could apply. + if (NotifierDeliveryStall?.Invoke(node) is { IsCompleted: false } stall) + { + MarkDeferred(e); + await stall.ConfigureAwait(false); + } + + // Re-checked: a stall may have outlived the condition (the liveness check above ran before it). + if (IsLiveCondition(condition)) DeliverToNotifierItems(monitored, e); + } + else + { + // Not a fan-out of ours (none exists today): the SDK path, which may suspend. Whatever is delivered after + // the suspension runs off Lock, so it must be judged as a deferred event (#855). + var pending = handler(SystemContext, node, e, CancellationToken.None); + if (!pending.IsCompleted) MarkDeferred(e); + await pending.ConfigureAwait(false); + } + } var notifiers = new List(); node.GetNotifiers(SystemContext, notifiers); @@ -1631,6 +1661,81 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 await DeliverConditionEventAsync(condition, parent, e).ConfigureAwait(false); } + /// Marks an event whose remaining delivery will run off Lock (#855) — before the walk suspends. + private static void MarkDeferred(IFilterTarget e) + { + if (e is ConditionEventSnapshot snapshot) snapshot.Deferred = true; + } + + /// + /// The SDK serving event items on , when that node is one of + /// ours and is the fan-out's own OnReportEventAsync (what + /// MonitoredNode2.Add(IEventMonitoredItem) installs); otherwise null (Gitea #855). + /// + private MonitoredNode2? OwnNotifierFanOut(NodeState node, NodeStateReportEventAsyncHandler handler) => + handler.Target is MonitoredNode2 monitored && ReferenceEquals(monitored.Node, node) && OwnsNamespace(node.NodeId) + ? monitored + : null; + + /// + /// Test probe (#855): consulted before a condition event is delivered to the items of one of our notifiers; an + /// incomplete task is awaited first, so the walk suspends there exactly as it would behind a slow notifier — the + /// way tests produce a deferral now that no notifier of ours can block. Null in production. + /// + internal Func? NotifierDeliveryStall { get; set; } + + /// + /// Delivers one condition event to the event items attached to one of this manager's notifiers (an equipment or + /// raw device folder), item by item (Gitea #855) — the folder twin of . + /// + /// The notifier's SDK fan-out; only its attached-item list is used. + /// The condition's snapshot. + /// + /// + /// Mirrors the SDK's MonitoredNode2.ProcessEventForItemAsync — its audit guard, then the verdict through + /// (the SDK's type + SourceNode ReceiveEvents + /// check and, under enforcement, #826's live-notifier + grant check), then QueueEvent — but per event: + /// nothing is memoised, so a deploy, revocation or reference removal decides the very next event. The item's + /// gate and refresh cut apply exactly as on the Server object (). + /// + /// + /// Synchronous. On a report it runs on the reporter's thread under Lock (as the Server-object delivery + /// does), and QueueEvent never waits for a consumer — the item's own queue discards oldest at its + /// QueueSize — so no notifier of ours defers any more. Off Lock (a deferred event) the verdict, + /// which takes Lock, is computed before the item gate is taken: nothing under a gate takes Lock. + /// A stood-down item (#562) is not in the attached list and receives nothing, as before. + /// + /// + private void DeliverToNotifierItems(MonitoredNode2 monitored, IFilterTarget e) + { + var deferred = e is ConditionEventSnapshot { Deferred: true } d ? d : null; + foreach (var item in monitored.EventMonitoredItems.Values) + { + if (item is null) continue; + try + { + if (IsAuditEvent(e) && !IsAuditDeliverableTo(item)) continue; + if (ServiceResult.IsBad(((INodeManager3)this).ValidateEventRolePermissions(item, e))) continue; + QueueConditionEventToItem(item, e, deferred, DeferredFolderDeliveryProbe); + } + catch (Exception ex) + { + // As the SDK's per-item delivery: one item's failure must not cost the others (or the rest of the walk). + _logger.LogWarning(ex, "OtOpcUaNodeManager: failed to deliver a condition event to event item {ItemId}", item.Id); + } + } + } + + /// The SDK's audit test from MonitoredNode2.ProcessEventForItemAsync: the event, or the snapshot's handle. + private static bool IsAuditEvent(IFilterTarget e) => + e is AuditEventState || e is InstanceStateSnapshot { Handle: AuditEventState }; + + /// + /// Test probe (#855): the folder twin of — runs between a deferred + /// event's supersession check and its queueing to a folder item, inside the item's gate. Null in production. + /// + internal Action? DeferredFolderDeliveryProbe { get; set; } + /// Whether is the instance the address space currently holds for its NodeId. private bool IsLiveCondition(NodeState condition) => _alarmConditions.TryGetValue(MapKey(condition.NodeId), out var live) && ReferenceEquals(live, condition); @@ -1711,18 +1816,27 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 public bool Retain { get; } = retain; + private volatile bool _deferred; + /// - /// Set (under the event-report gate, before the drain can dequeue it) when the event waits in a deferral queue: - /// only such an event reaches Server-object items off Lock, which is the only way a refresh can be overtaken. + /// Set when the event's delivery continues off Lock: under the event-report gate, before the drain can + /// dequeue it, when it waits in a deferral queue; or by the walk itself, before it suspends at a hop (#855), for + /// the in-flight event whose remaining hops then run on another thread. Only such an event reaches an item off + /// Lock, which is the only way a refresh can be overtaken. /// - public bool Deferred { get; set; } + public bool Deferred + { + get => _deferred; + set => _deferred = value; + } } /// - /// Per Server-object item state (#854): the gate that orders a deferred delivery to the item against a refresh of - /// the item, and the cut of the item's latest refresh while deferred events may still be older than it. + /// Per event item state — Server-object (#854) and, since #855, folder items: the gate that orders a deferred + /// delivery to the item against a refresh of the item, and the cut of the item's latest refresh while deferred + /// events may still be older than it. /// - private sealed class AllEventsItemState + private sealed class EventItemState { /// Held across "is this deferred event superseded? → QueueEvent", and across a refresh's /// "QueueEvent the replay → install the cut". Only ever taken innermost (nothing but the item's own lock under it). @@ -1749,21 +1863,21 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 private int _refreshGeneration; /// - /// One per Server-object item for the item's whole lifetime (#854 review): keyed by + /// One per event item for the item's whole lifetime (#854 review): keyed by /// the item object itself, so an unsubscribe + resubscribe of the same item between a refresh and the drain keeps its /// cut and its gate, and a drain holding the state always holds the one a refresh installs into. /// - private readonly ConditionalWeakTable _allEventsItemStates = new(); + private readonly ConditionalWeakTable _eventItemStates = new(); - /// Server-object items currently holding a refresh cut (test observation, #854 review). + /// Server-object and folder event items currently holding a refresh cut (test observation, #854 review). internal int ActiveRefreshCuts { get { var n = 0; - foreach (var item in _allEventsItems.Keys) + foreach (var item in TrackedEventItems()) { - if (!_allEventsItemStates.TryGetValue(item, out var state)) continue; + if (!_eventItemStates.TryGetValue(item, out var state)) continue; lock (state.Gate) n += state.Cut is null ? 0 : 1; } @@ -1771,8 +1885,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } } - private AllEventsItemState AllEventsItemStateOf(IEventMonitoredItem item) => - _allEventsItemStates.GetValue(item, static _ => new AllEventsItemState()); + private EventItemState EventItemStateOf(IEventMonitoredItem item) => + _eventItemStates.GetValue(item, static _ => new EventItemState()); + + /// + /// Every live event item this manager tracks: the Server-object items and the direct (folder) items (#855). An + /// item that left both maps keeps its state in until collected, but any cut it holds + /// is from an ended episode by the time it could deliver again, and so inert. + /// + private IEnumerable TrackedEventItems() => _allEventsItems.Keys.Concat(_eventItems.Keys); /// /// Advanced (under the event-report gate) each time the deferral count returns to zero: the end of a deferral @@ -1810,9 +1931,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 private void ClearStaleRefreshCuts() { var current = Interlocked.Read(ref _deferralEpisode); - foreach (var item in _allEventsItems.Keys) + foreach (var item in TrackedEventItems()) { - if (!_allEventsItemStates.TryGetValue(item, out var state)) continue; + if (!_eventItemStates.TryGetValue(item, out var state)) continue; lock (state.Gate) { if (state.Cut is { } cut && cut.Episode < current) state.Cut = null; // episodes only grow: never clear a newer cut (#854 re-review) @@ -5671,38 +5792,51 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 continue; } - // #854: a synchronous event (under Lock) needs the gate only while a refresh's replay is pending, and the - // pending marker is installed under Lock, so an item no refresh has touched has nothing to order. A deferred - // event always takes the gate — its state is created here if need be, so a refresh racing it shares it. - AllEventsItemState? state; - if (deferred is not null) state = AllEventsItemStateOf(item); - else if (!_allEventsItemStates.TryGetValue(item, out state) || state.PendingReplay == 0 || e is not ConditionEventSnapshot) + QueueConditionEventToItem(item, e, deferred, DeferredServerObjectDeliveryProbe); + } + } + + /// + /// Queues a condition event to one event item — Server-object or folder (#855) — under the item's refresh gate and + /// cut (#854). The caller has already decided the item may receive it, and holds no item gate. + /// + /// The item. + /// The event. + /// The event, when its delivery runs off Lock (). + /// Test probe run inside the gate, before queueing a deferred event. + private void QueueConditionEventToItem(IEventMonitoredItem item, IFilterTarget e, ConditionEventSnapshot? deferred, Action? probe) + { + // #854: a synchronous event (under Lock) needs the gate only while a refresh's replay is pending, and the + // pending marker is installed under Lock, so an item no refresh has touched has nothing to order. A deferred + // event always takes the gate — its state is created here if need be, so a refresh racing it shares it. + EventItemState? state; + if (deferred is not null) state = EventItemStateOf(item); + else if (!_eventItemStates.TryGetValue(item, out state) || state.PendingReplay == 0 || e is not ConditionEventSnapshot) + { + item.QueueEvent(e); + return; + } + + // A deferred event reaches this item off Lock, so a ConditionRefresh of the item may have replayed a newer + // state since it was reported; and while a refresh's frozen replay is being queued, an event newer than it must + // not be followed by it. Both are decided and queued under the item's gate, which the refresh also holds for + // each replayed event it queues — so a deferred event lands before the replay or is judged against the cut. + lock (state.Gate) + { + // A cut from an episode that has ended is stale (only waiting for ClearStaleRefreshCuts). + if (deferred is not null + && state.Cut is { } cut && cut.Episode == Interlocked.Read(ref _deferralEpisode) && cut.Supersedes(deferred)) { - item.QueueEvent(e); - continue; + Interlocked.Increment(ref _refreshSupersededDeliveries); + OtOpcUaTelemetry.ConditionEventDeliveriesSuperseded.Add(1); + return; } - // A deferred event reaches this item off Lock, so a ConditionRefresh of the item may have replayed a newer - // state since it was reported; and while a refresh's frozen replay is being queued, an event newer than it must - // not be followed by it. Both are decided and queued under the item's gate, which the refresh also holds for - // each replayed event it queues — so a deferred event lands before the replay or is judged against the cut. - lock (state.Gate) - { - // A cut from an episode that has ended is stale (only waiting for ClearStaleRefreshCuts). - if (deferred is not null - && state.Cut is { } cut && cut.Episode == Interlocked.Read(ref _deferralEpisode) && cut.Supersedes(deferred)) - { - Interlocked.Increment(ref _refreshSupersededDeliveries); - OtOpcUaTelemetry.ConditionEventDeliveriesSuperseded.Add(1); - continue; - } + if (state.PendingReplay != 0 && e is ConditionEventSnapshot { } ce && ce.Sequence > state.PendingWatermark) + state.NewerSinceCut.Add(ce.ConditionId); - if (state.PendingReplay != 0 && e is ConditionEventSnapshot { } ce && ce.Sequence > state.PendingWatermark) - state.NewerSinceCut.Add(ce.ConditionId); - - if (deferred is not null) DeferredServerObjectDeliveryProbe?.Invoke(); - item.QueueEvent(e); - } + if (deferred is not null) probe?.Invoke(); + item.QueueEvent(e); } } @@ -6080,8 +6214,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// would be. /// /// - /// Runs on the SDK's per-node delivery task, where the inherited role-permission check already takes - /// Lock (via GetNodeMetadata); the scope resolution here takes it the same way. + /// Runs per event, uncached, from the per-item folder delivery (#855) — on the reporter's thread under + /// Lock, or off it for a deferred event (before the item gate is taken) — and, for the events that still + /// travel the SDK channel (a condition's method audits), on the SDK's per-node delivery task, where the + /// inherited role-permission check already takes Lock (via GetNodeMetadata); the scope + /// resolution here takes it the same way. /// /// private bool IsNotifierEventVisibleTo(IEventMonitoredItem item, IFilterTarget e) @@ -6146,13 +6283,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// Good to deliver; otherwise the veto. /// /// - /// The SDK's MonitoredNode2 asks this, per event and per item, before it queues an event to an item - /// attached to one of this manager's notifiers. The Server-object path (#586) and ConditionRefresh decide - /// the same question themselves, so a folder's subscribers stop receiving a moved alarm (or a native alarm - /// whose reference was dropped) the moment the generation says so, not when the rebuild lands. + /// Condition events to an item attached to one of this manager's notifiers are decided here per event and per + /// item by (#855, uncached); the SDK's MonitoredNode2 still asks it + /// for what travels its channel (a condition's method audit events). The Server-object path (#586) and + /// ConditionRefresh decide the same question themselves, so a folder's subscribers stop receiving a moved + /// alarm (or a native alarm whose reference was dropped) the moment the generation says so, not when the + /// rebuild lands. /// /// - /// ⚠️ The SDK MEMOIZES the verdict per (item, event type, SourceNode) until the monitored node's + /// ⚠️ On the SDK channel path the SDK MEMOIZES the verdict per (item, event type, SourceNode) until the monitored node's /// permission generation moves, so must run whenever the inputs /// change: a new authority generation or authorizer, the revalidation tick, and the end of every apply. A /// condition's SourceNode is the condition itself, so a verdict is never shared between two conditions. @@ -6278,9 +6417,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 // client connecting after a transition gets the current alarm picture. Those items are always refreshed // here (RefreshServerObjectItems): with enforcement on through the same IsConditionVisibleTo decision live // delivery uses, and in both modes so the refresh can cut off deferred events it supersedes (#854). - // Items on one of this manager's own notifiers keep the base behaviour with enforcement off (a stood-down - // item is off its notifier's fan-out, which the base already treats as "not refreshed"), and - // RefreshDirectItems with it on. + // Items on one of this manager's own notifiers go through RefreshDirectItems in both modes since #855 — the + // base's loop (a stood-down item is off its notifier's fan-out, so not refreshed) plus the #826 decision with + // enforcement on, and the same cut against deferred events as the Server object. /// public override ServiceResult ConditionRefresh(OperationContext context, IList monitoredItems) { @@ -6301,11 +6440,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 var enforced = AclEnforced; var systemContext = SystemContext.Copy(context); - if (direct.Count > 0) - { - if (enforced) RefreshDirectItems(systemContext, direct); - else base.ConditionRefresh(context, direct); - } + if (direct.Count > 0) RefreshDirectItems(systemContext, direct, enforced); if (serverItems.Count > 0) RefreshServerObjectItems(systemContext, serverItems, enforced); return ServiceResult.Good; @@ -6339,7 +6474,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// in its deferral queue (#838) and reach Server-object items only when the drain gets to them, off /// Lock. A refresh in that window replays the condition's CURRENT state between RefreshStart and /// RefreshEnd; the waiting events, all older, then followed RefreshEnd — a stale state, or a condition the - /// refresh had just dropped re-added, until the newest drained. + /// refresh had just dropped re-added, until the newest drained. (Since #855 folder items are no longer a + /// channel, so nothing of ours defers today; the cut stays as the safety net, and folder items share it — + /// .) /// /// /// Common case — nothing deferred: unchanged, and off Lock. The replay is the notifier walk the SDK @@ -6353,10 +6490,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// Deferring — the cut. In ONE hold of Lock (under which every condition event is reported): the /// watermark (), the episode, and a frozen snapshot of each Retain /// condition read from (O(conditions), no tree walk), then queued to each item - /// with its installed under the item's . The ACL / + /// with its installed under the item's . The ACL / /// role verdicts are computed BEFORE taking Lock, against the live conditions (they depend on the event /// type and source, not state). The cut and a "replay pending" marker are installed per item under its - /// inside that Lock hold; the replay is then queued OFF + /// inside that Lock hold; the replay is then queued OFF /// Lock, one event per hold of the gate. While the marker is set every condition event to the item takes /// the gate: one newer than the cut marks its condition so its (older) frozen replay entry is dropped instead of /// following it; a deferred one at or below the cut is judged against the cut () @@ -6406,14 +6543,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 { // A cut-path replay of this item still being queued by a concurrent refresh is older than this live one: // stop it, so it cannot follow this replay with frozen states. - if (_allEventsItemStates.TryGetValue(item, out var pending)) - { - lock (pending.Gate) - { - pending.PendingReplay = 0; - pending.NewerSinceCut.Clear(); - } - } + StopPendingReplay(item); foreach (var (e, _, scopes) in replay) { @@ -6472,7 +6602,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } } - var states = items.Select(AllEventsItemStateOf).ToList(); + var states = items.Select(EventItemStateOf).ToList(); var perItem = new List<(IFilterTarget Event, AlarmConditionState Condition)>[items.Count]; var generation = Interlocked.Increment(ref _refreshGeneration); // #854 re-review: a throw between installing the "replay pending" markers and finishing the replay must not leave @@ -6540,20 +6670,34 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } } - // Queued off Lock, one replayed event per hold of the item gate: an event that reached the item since the cut is - // newer than the frozen replay of its condition, so that replay entry is dropped rather than queued after it (the - // newer event is already inside RefreshStart..RefreshEnd). A later refresh of the item supersedes this one. ServerObjectRefreshProbe?.Invoke(true, Monitor.IsEntered(Lock)); + QueueFrozenReplay(items, states, perItem.Select(entries => entries.Select(x => (x.Event, x.Condition.NodeId)).ToList()).ToArray(), generation); + } + finally + { + ClearPendingReplays(states, generation); + } + } + + /// + /// Queues a cut-path refresh's frozen replay to its items OFF Lock, one replayed event per hold of the item + /// gate (#854; folder items too since #855): an event that reached the item since the cut is newer than the frozen + /// replay of its condition, so that replay entry is dropped rather than queued after it (the newer event is already + /// inside RefreshStart..RefreshEnd). A later refresh of the item supersedes this one. + /// + private static void QueueFrozenReplay( + List items, List states, List<(IFilterTarget Event, NodeId ConditionId)>[] perItem, int generation) + { for (var i = 0; i < items.Count; i++) { var item = items[i]; var state = states[i]; - foreach (var (e, condition) in perItem[i]) + foreach (var (e, conditionId) in perItem[i]) { lock (state.Gate) { if (state.PendingReplay != generation) break; - if (state.NewerSinceCut.Contains(condition.NodeId)) continue; + if (state.NewerSinceCut.Contains(conditionId)) continue; item.QueueEvent(e); } } @@ -6567,51 +6711,227 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } } } - } - finally + } + + /// + /// #854 re-review: a throw between installing the "replay pending" markers and finishing the replay must not leave a + /// marker set (every sync event to that item would keep taking the gate until its next refresh). + /// + private static void ClearPendingReplays(List states, int generation) + { + foreach (var state in states) { - foreach (var state in states) + lock (state.Gate) { - lock (state.Gate) - { - if (state.PendingReplay != generation) continue; - state.PendingReplay = 0; - state.NewerSinceCut.Clear(); - } + if (state.PendingReplay != generation) continue; + state.PendingReplay = 0; + state.NewerSinceCut.Clear(); } } } /// - /// ConditionRefresh for event items on this manager's own notifiers, under enforcement (Gitea #826): the SDK - /// base's loop exactly — an item no longer attached (stood down by revalidation, or tombstoned) is not - /// refreshed — plus the per-event decision live delivery makes (), so a - /// refresh cannot replay to a folder's subscriber an alarm that folder's live events withhold. + /// Stops a cut-path replay of still being queued by a concurrent refresh: a live replay that + /// is about to be queued is newer, and the frozen one must not follow it. + /// + private void StopPendingReplay(IEventMonitoredItem item) + { + if (!_eventItemStates.TryGetValue(item, out var pending)) return; + lock (pending.Gate) + { + pending.PendingReplay = 0; + pending.NewerSinceCut.Clear(); + } + } + + /// + /// ConditionRefresh (and ConditionRefresh2) for event items on this manager's own notifiers — equipment and raw device + /// folders — flag on and off: never overtaken by a deferred event older than what it replays (Gitea #855), the folder + /// twin of . /// /// The refresh's system context. /// The direct (non-Server-object) items being refreshed. - private void RefreshDirectItems(ServerSystemContext systemContext, List items) + /// Whether the data-plane ACL is enforced. + /// + /// + /// Nothing deferred (the only case today — no notifier of ours defers since #855): unchanged, off + /// Lock. The SDK base's loop — an item no longer attached (stood down by revalidation, or tombstoned) + /// is not refreshed; the item's notifier is walked and each live condition queued through the SDK's role check — + /// plus, under enforcement, the per-event decision live delivery makes (, + /// #826), so a refresh cannot replay to a folder's subscriber an alarm that folder's live events withhold. It + /// stands only if no deferral episode existed at any moment of it; the closing check takes Lock once, so a + /// report whose walk suspended (it marks the event before suspending, under Lock) has been counted by then. + /// Otherwise the cut path runs as well, and its replay — later inside the same RefreshStart..RefreshEnd — + /// supersedes anything that slipped in between. + /// + /// + /// Deferring — the cut, exactly as on the Server object: verdicts per (item, condition) first, off + /// Lock, from a live walk; then one Lock hold takes the watermark, the episode, the Retain set and, + /// per item, the same walk as frozen snapshots, and installs the cut and the "replay pending" marker under the + /// item's gate; the replay is queued off Lock (). Deferred events reaching + /// the item afterwards are judged against the cut (). + /// + /// + private void RefreshDirectItems(ServerSystemContext systemContext, List items, bool enforced) { - foreach (var item in items) + long episodeBefore; + bool deferringBefore; + lock (_eventReportGate) { - NodeState? notifier; - lock (Lock) + episodeBefore = _deferralEpisode; + deferringBefore = _deferredEventReports > 0; + } + + if (!deferringBefore) + { + foreach (var item in items) { - if (!MonitoredItems.ContainsKey(item.Id)) continue; - notifier = (item.ManagerHandle as NodeHandle)?.Node; + var notifier = AttachedNotifierOf(item); + if (notifier is null) continue; + + StopPendingReplay(item); + var events = new List(); + notifier.ConditionRefresh(systemContext, events, true); + DirectRefreshProbe?.Invoke(false, Monitor.IsEntered(Lock)); + foreach (var e in events) + { + if (!IsDirectRefreshEventAllowed(item, e, enforced)) continue; + item.QueueEvent(e); + } } - if (notifier is null) continue; + // Lock first: a report that suspended under it has registered its deferral by the time Lock is free. + lock (Lock) + { + lock (_eventReportGate) + { + if (_deferralEpisode == episodeBefore && _deferredEventReports == 0) return; + } + } + } + RefreshDirectItemsWithCut(systemContext, items, enforced); + } + + /// The notifier an attached direct item subscribes to, or null for one that is not attached (not refreshed). + private NodeState? AttachedNotifierOf(IEventMonitoredItem item) + { + lock (Lock) + { + return MonitoredItems.ContainsKey(item.Id) ? (item.ManagerHandle as NodeHandle)?.Node : null; + } + } + + /// A direct item's refresh verdict for one replayed event: the #826 decision under enforcement, then the SDK's. + private bool IsDirectRefreshEventAllowed(IEventMonitoredItem item, IFilterTarget e, bool enforced) => + (!enforced || IsNotifierEventVisibleTo(item, e)) && !ServiceResult.IsBad(ValidateEventRolePermissions(item, e)); + + /// + /// Test probe (#855): called by a direct-item refresh with whether it took the cut path and whether the calling + /// thread held Lock while it queued the replay. Null in production. + /// + internal Action? DirectRefreshProbe { get; set; } + + /// The cut path of — see there. + private void RefreshDirectItemsWithCut(ServerSystemContext systemContext, List items, bool enforced) + { + // Verdicts per (item, condition) before Lock, from a live walk: they depend on the event type, source, notifier + // wiring and grants, not on the condition's state. + var allowed = new HashSet<(IEventMonitoredItem, NodeState)>(); + var known = new HashSet(System.Collections.Generic.ReferenceEqualityComparer.Instance); + foreach (var item in items) + { + if (AttachedNotifierOf(item) is not { } notifier) continue; var events = new List(); notifier.ConditionRefresh(systemContext, events, true); foreach (var e in events) { - if (!IsNotifierEventVisibleTo(item, e)) continue; - if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue; - item.QueueEvent(e); + if (ConditionOf(e) is not { } node) continue; + known.Add(node); + if (IsDirectRefreshEventAllowed(item, e, enforced)) allowed.Add((item, node)); } } + + var states = items.Select(EventItemStateOf).ToList(); + var perItem = new List<(IFilterTarget Event, NodeId ConditionId)>[items.Count]; + var generation = Interlocked.Increment(ref _refreshGeneration); + try + { + lock (Lock) + { + var watermark = Interlocked.Read(ref _conditionEventSequence); + long episode; + bool deferring; + lock (_eventReportGate) + { + episode = _deferralEpisode; + deferring = _deferredEventReports > 0; + } + + var retained = new HashSet(); + foreach (var condition in _alarmConditions.Values) + { + if (condition.Retain?.Value == true) retained.Add(condition.NodeId); + } + + for (var i = 0; i < items.Count; i++) + { + var item = items[i]; + var entries = new List<(IFilterTarget Event, NodeId ConditionId)>(); + perItem[i] = entries; + var state = states[i]; + + // Lock is held, so this is the attachment the walk below sees. An unattached item is not refreshed and + // gets no cut (it receives nothing until re-attached, and then every event is decided as it arrives). + var notifier = MonitoredItems.ContainsKey(item.Id) ? (item.ManagerHandle as NodeHandle)?.Node : null; + if (notifier is null) + { + lock (state.Gate) + { + state.PendingReplay = 0; + state.NewerSinceCut.Clear(); + } + + continue; + } + + // Frozen at the watermark: the replay is exactly the state every event at or below it describes. + var live = new List(); + notifier.ConditionRefresh(SystemContext, live, true); + var refreshed = new HashSet(); + foreach (var target in live) + { + if (target is not BaseInstanceState instance) continue; + // A condition materialised since the verdicts were taken is decided here (rare). + if (!allowed.Contains((item, instance))) + { + if (known.Contains(instance)) continue; + if (!IsDirectRefreshEventAllowed(item, target, enforced)) continue; + } + + var snapshot = new InstanceStateSnapshot(); + snapshot.Initialize(SystemContext, instance); + entries.Add((snapshot, instance.NodeId)); + refreshed.Add(instance.NodeId); + } + + lock (state.Gate) + { + state.Cut = deferring ? new RefreshCut(watermark, episode, refreshed, retained) : null; + state.PendingWatermark = watermark; + state.NewerSinceCut.Clear(); + state.PendingReplay = generation; + } + } + } + + DirectRefreshProbe?.Invoke(true, Monitor.IsEntered(Lock)); + QueueFrozenReplay(items, states, perItem, generation); + } + finally + { + ClearPendingReplays(states, generation); + } } /// The condition a refresh event describes, or null when it names none (withheld: fail closed). diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs index 18cc6d003..bba4a23eb 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs @@ -24,16 +24,18 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// every other subscriber still receives them, and the refresh never waits for the stalled drain. /// /// -/// The overload is produced deterministically: the folder item's server-side MonitoredItem lock is held, so -/// the SDK's per-node delivery task parks in QueueEvent, its 4096-slot channel fills, and the condition's -/// later events are deferred — without holding the node manager's Lock, which the refresh needs. +/// The overload is produced deterministically: delivery is stalled at the EqA folder (), so the +/// condition's walk suspends there and its later events are deferred — without holding the node manager's Lock, +/// which the refresh needs. (Before Gitea #855 this held the folder item's SDK lock so the folder's 4096-slot channel +/// filled; folder items are now queued per item by the reporter, and no notifier of ours defers on its own.) The +/// folder-item twin of these tests is . /// public sealed class ConditionRefreshDeferralWireTests { private const string EqA = ServerEventRig.EqA; - // > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces. - private const int FloodSize = 5000; + // Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces. + private const int FloodSize = 500; /// Per-folder / Server-object ACL on or off (off is the production default). /// @@ -84,7 +86,7 @@ public sealed class ConditionRefreshDeferralWireTests await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed")); await folderSubscription.SetPublishingModeAsync(false, ct); - using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id))) + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events"); @@ -200,7 +202,7 @@ public sealed class ConditionRefreshDeferralWireTests serverEvents.Has(onServer, "seed").ShouldBeFalse("the key holds no grant yet"); await folderSubscription.SetPublishingModeAsync(false, ct); - using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id))) + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); rig.Project(native: true, Active("final")); @@ -271,7 +273,7 @@ public sealed class ConditionRefreshDeferralWireTests rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw); }; - using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id))) + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred"); @@ -345,7 +347,7 @@ public sealed class ConditionRefreshDeferralWireTests proceed.Wait(TimeSpan.FromSeconds(30)); }; - using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id))) + using (NotifierStall.Hold(rig.Nm, EqA)) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); rig.Project(native: true, Active("final")); @@ -416,7 +418,6 @@ public sealed class ConditionRefreshDeferralWireTests rig.Project(native: true, Active("s0")); await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0")); await folderSubscription.SetPublishingModeAsync(false, ct); - var item = FindServerItem(rig.Nm, onA.Status.Id); using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct); var counter = 0; @@ -440,7 +441,7 @@ public sealed class ConditionRefreshDeferralWireTests { while (!stop.IsCancellationRequested) { - using (ItemLockStall.Hold(item)) + using (NotifierStall.Hold(rig.Nm, EqA)) { Thread.Sleep(400); if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen); @@ -568,7 +569,7 @@ public sealed class ConditionRefreshDeferralWireTests await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed")); await folderSubscription.SetPublishingModeAsync(false, ct); - var stall = deferring ? ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)) : null; + var stall = deferring ? NotifierStall.Hold(rig.Nm, EqA) : null; try { if (deferring) @@ -622,7 +623,7 @@ public sealed class ConditionRefreshDeferralWireTests } /// The server-side Server-object (all-events) item behind a client item. - private static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id) + internal static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id) { var map = (IDictionary)typeof(OtOpcUaNodeManager) .GetField("_allEventsItems", BindingFlags.Instance | BindingFlags.NonPublic)! @@ -636,7 +637,7 @@ public sealed class ConditionRefreshDeferralWireTests } /// The server-side event item behind a client item, from the node manager's direct-item map. - private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id) + internal static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id) { var map = (IDictionary)typeof(OtOpcUaNodeManager) .GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)! @@ -664,45 +665,4 @@ public sealed class ConditionRefreshDeferralWireTests ScopeId = $"eq-{equipment}", PermissionFlags = NodePermissions.ReadOnly, }; - - /// - /// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task - /// parks in QueueEvent for that item (a stalled subscriber) without the node manager's Lock held. - /// - private sealed class ItemLockStall : IDisposable - { - private readonly ManualResetEventSlim _release = new(); - private readonly Thread _thread; - - private ItemLockStall(IEventMonitoredItem item) - { - var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem) - .GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)! - .GetValue(item)!; - using var held = new ManualResetEventSlim(); - _thread = new Thread(() => - { - gate.Enter(); - try - { - held.Set(); - _release.Wait(); - } - finally { gate.Exit(); } - }) { IsBackground = true }; - _thread.Start(); - held.Wait(); - } - - public static ItemLockStall Hold(IEventMonitoredItem item) => new(item); - - public void Release() => _release.Set(); - - public void Dispose() - { - _release.Set(); - _thread.Join(); - _release.Dispose(); - } - } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs index 1ca3b158a..81039208d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs @@ -27,13 +27,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// no ACL-denied event reaches an item during the overflow, including deferred events whose grant was /// revoked while they waited. /// +/// Since Gitea #855 the folder items are served per item by the reporter (as the Server-object items always were), +/// so the flood no longer defers at all — the first test pins that — and the revocation test stalls the folder's +/// delivery () to get a deferred backlog. /// public sealed class EventQueueOverflowAclWireTests { private const string EqA = ServerEventRig.EqA; private const string EqB = ServerEventRig.EqB; - // Native events per flood: past the SDK's 4096-slot per-notifier channel, so the native condition defers. + // Native events per flood: past the SDK's 4096-slot per-notifier channel (which deferred them before #855). private const int PerCondition = 6000; /// @@ -150,13 +153,12 @@ public sealed class EventQueueOverflowAclWireTests readLatencies.Max().ShouldBeLessThan(TimeSpan.FromSeconds(5), "a Read must not queue behind the whole flood"); (await TimedReadAsync(anonymous, eqA, ct)).ShouldBeLessThan(TimeSpan.FromSeconds(2), "the node manager is free afterwards"); - // Memory: the node manager's own overflow store stayed within its per-condition budget (queued + in flight)… - Volatile.Read(ref peakDeferred).ShouldBeLessThanOrEqualTo( - 2 * (OtOpcUaNodeManager.MaxDeferredEventsPerCondition + 1), "deferral is bounded per condition"); - if (singleHold) rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the native tail overflowed and coalesced"); + // Memory: the node manager held nothing of its own for the overflow (#855: no notifier of ours defers, so the + // item queues are the only store, bounded below)… + Volatile.Read(ref peakDeferred).ShouldBe(0, "folder items are served per item by the reporter: nothing defers"); + rig.Nm.DiscardedEventReports.ShouldBe(0); - // …let the SDK channel finish draining into the (still unpublished) item queues, then publish. - await Task.Delay(TimeSpan.FromSeconds(2), ct); + // …then publish the (still unpublished) item queues. await keySub.SetPublishingModeAsync(true, ct); var lastNative = $"n{PerCondition - 1}"; await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(onB, lastNative) && keyEvents.Has(keyServer, lastNative)); @@ -165,10 +167,9 @@ public sealed class EventQueueOverflowAclWireTests foreach (var item in new[] { onB, keyServer }) { var received = keyEvents.Of(item); - // ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (Under a single hold the - // deferred native tail drains last and evicts every synchronously-queued event from the 1000-slot queue, so - // there this check cannot see a leak; the "after-s" leg below is the falsifier for that mode — verified by - // running with enforcement off: chunked fails here, single-hold fails at "after-s".) + // ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (The 1000-slot queue keeps + // only the last events of the flood, which interleave n/s, so a leak shows here in both modes; the + // "after-s" leg below is a second falsifier.) received.Where(f => RecordedEvents.Message(f)?.StartsWith('s') == true) .ShouldBeEmpty($"{item.DisplayName}: an ACL-denied event leaked during the overflow"); @@ -211,7 +212,7 @@ public sealed class EventQueueOverflowAclWireTests } /// - /// A grant revoked while a deferred backlog waits behind a full notifier channel: the deferred events are decided + /// A grant revoked while a deferred backlog waits behind a stalled notifier: the deferred events are decided /// at delivery against the CURRENT generation, so none of them — nor any event reported after the revocation — /// reaches the revoked session's folder or Server-object item, while a still-granted session sharing the same /// folder channel receives the whole tail (positive control). @@ -249,6 +250,9 @@ public sealed class EventQueueOverflowAclWireTests await anonSub.ApplyChangesAsync(ct); anonA.Status.Error.ShouldBeNull(); + // The EqA folder's delivery stalls (#855: nothing of ours defers on its own any more), so the native condition's + // walk suspends there and the rest of the flood is deferred behind it. + using var stall = NotifierStall.Hold(rig.Nm, EqA); var flood = Task.Factory.StartNew(() => { lock (rig.Nm.Lock) @@ -276,11 +280,12 @@ public sealed class EventQueueOverflowAclWireTests throw new ShouldAssertException("revoking a grant under Lock while events were deferred stalled (deadlock)"); } + stall.Release(); await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); await Task.Delay(TimeSpan.FromSeconds(2), ct); await keySub.SetPublishingModeAsync(true, ct); - // Positive control: the still-granted session on the SAME folder channel gets the whole deferred tail. + // Positive control: the still-granted session on the SAME folder gets the whole deferred tail. var lastNative = $"n{PerCondition - 1}"; await RecordedEvents.WaitUntilAsync(() => anonEvents.Has(anonA, lastNative) && anonEvents.Has(anonA, "after-revoke") diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs new file mode 100644 index 000000000..b01a93427 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs @@ -0,0 +1,627 @@ +using System.Diagnostics; +using System.Globalization; +using Opc.Ua; +using Opc.Ua.Client; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using MonitoredItem = Opc.Ua.Client.MonitoredItem; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; + +/// +/// Gitea #855: a ConditionRefresh of an event item on an equipment / raw device FOLDER gets the guarantee the +/// Server object got in #854 — no condition event older than what the refresh replayed reaches the item after +/// RefreshEnd, per-condition order is kept, the per-folder ACL (#826) decides every event and every replayed condition, +/// and neither a reporter nor the refresh ever waits on a stalled delivery. +/// +/// +/// +/// Folder items are now served per item by the node manager (no SDK channel), so in production no notifier defers +/// at all; pins that under a +/// flood with concurrent refreshes. Every other test produces the deferral the cut exists for by stalling delivery +/// at a notifier (), the way a full SDK channel used to. +/// +/// +/// The rig's native condition is wired into EqA and EqB and hangs under the raw device folder, so its walk is +/// condition → EqA → … → device folder; a stall at EqA suspends it before EqA's items and every later hop. +/// +/// +public sealed class FolderConditionRefreshWireTests +{ + private const string EqA = ServerEventRig.EqA; + + // Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces. + private const int FloodSize = 500; + + /// Per-folder ACL on or off (off is the production default). + /// Whether the condition is still Retain at refresh time (re-sent) or cleared (omitted). + /// ConditionRefresh2 on the folder item instead of ConditionRefresh on its subscription. + /// + /// The folder item is modified (ModifyMonitoredItems re-subscribes it to its notifier) between the refresh and the + /// drain: its cut must survive, as the Server object's does across an unsubscribe + resubscribe. + /// + [Theory] + [InlineData(false, true, false, false)] + [InlineData(true, true, false, false)] + [InlineData(false, false, false, false)] + [InlineData(true, false, false, false)] + [InlineData(false, true, true, false)] + [InlineData(true, false, true, false)] + [InlineData(false, true, false, true)] + [InlineData(true, false, false, true)] + public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_folder_item_after_RefreshEnd( + bool enforced, bool finalRetained, bool refresh2, bool modify) + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + // A second, non-refreshing subscriber on the SAME folder: it must keep the whole deferred tail. + using var otherSession = await rig.ConnectAsync(anonymous: false, ct); + var otherEvents = new RecordedEvents(); + var otherSubscription = await RecordedEvents.SubscribeAsync(otherSession, otherEvents, ct); + var otherOnA = RecordedEvents.AddEventItem(otherSubscription, ServerEventRig.UnsNodeId(otherSession, EqA)); + await otherSubscription.ApplyChangesAsync(ct); + otherOnA.Status.Error.ShouldBeNull(); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA)); + await subscription.ApplyChangesAsync(ct); + onA.Status.Error.ShouldBeNull(); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed") && otherEvents.Has(otherOnA, "seed")); + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the stall must actually have deferred events"); + rig.Project(native: true, finalRetained ? Active("final") : Cleared("final")); + + var refresh = Stopwatch.StartNew(); + await RefreshAsync(session, subscription, onA, refresh2, ct); + refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled delivery"); + await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType)); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd"); + rig.Nm.ActiveRefreshCuts.ShouldBe(1, "only the refreshed folder item holds a cut"); + + if (modify) + { + onA.QueueSize = 1001; + await subscription.ApplyChangesAsync(ct); + onA.Status.Error.ShouldBeNull(); + } + + stall.Release(); + } + + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + await RecordedEvents.WaitUntilAsync(() => rig.Nm.ActiveRefreshCuts == 0); + rig.Project(native: true, Active("after")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after") && otherEvents.Has(otherOnA, "after")); + await Task.Delay(TimeSpan.FromMilliseconds(300), ct); + + var (replayed, afterEnd) = Window(events.Of(onA)); + if (finalRetained) + { + replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state"); + afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd"); + } + else + { + replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh"); + afterEnd.ShouldBe(["final", "after"], "stale Retain=true events would re-add the alarm; the Retain=false one agrees"); + events.Of(onA).Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse(); + } + + rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0); + + // The other subscriber on the same folder did not refresh: it keeps every deferred event, in order. + var flood = Indices(otherEvents.Of(otherOnA), 'f'); + flood.ShouldBe(Enumerable.Range(0, FloodSize).ToList(), "the non-refreshing folder subscriber keeps the whole tail, in order"); + otherEvents.Has(otherOnA, "final").ShouldBeTrue(); + } + + /// + /// The race the item gate closes, for a folder item: a deferred event is between its supersession check and its + /// queueing to the folder item when the refresh arrives. The refresh must wait for that one delivery, so it lands + /// before the replay rather than after RefreshEnd. + /// + [Fact] + public async Task A_folder_refresh_waits_for_a_deferred_delivery_already_past_its_check() + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Gate", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA)); + await subscription.ApplyChangesAsync(ct); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed")); + + using var entered = new ManualResetEventSlim(); + using var proceed = new ManualResetEventSlim(); + var probed = 0; + rig.Nm.DeferredFolderDeliveryProbe = () => + { + if (Interlocked.Increment(ref probed) != 1) return; + entered.Set(); + proceed.Wait(TimeSpan.FromSeconds(30)); + }; + + using (NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + rig.Project(native: true, Active("final")); + } + + // The released walk parks on its first deferred delivery to the folder item, past the check, inside the gate. + entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the folder item"); + var refresh = session.CallAsync( + ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + await Task.Delay(TimeSpan.FromSeconds(1), ct); + events.HasType(onA, ObjectTypeIds.RefreshEndEventType) + .ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate"); + proceed.Set(); + await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct); + await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType)); + rig.Nm.DeferredFolderDeliveryProbe = null; + + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + rig.Project(native: true, Active("after")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after")); + await Task.Delay(TimeSpan.FromMilliseconds(300), ct); + + var (window, afterEnd) = Window(events.Of(onA)); + window.Count.ShouldBe(2, "the parked delivery, then the replay"); + window[0]!.ShouldStartWith("f"); + window[1].ShouldBe("final"); + afterEnd.ShouldBe(["after"]); + } + + /// + /// The cut path queues its frozen replay OFF Lock. An event newer than the cut that reaches the folder item + /// in that window (a condition delivered synchronously, reported from the probe between the cut and the queueing) + /// must not be followed by its condition's older frozen replay entry. + /// + [Fact] + public async Task An_event_newer_than_the_folder_cut_is_never_followed_by_its_older_frozen_replay() + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Pending", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + // A second condition under the device folder only: its walk never reaches the stalled EqA, so it stays synchronous. + const string Free = ServerEventRig.RawDeviceFolder + "/free"; + rig.Sink.MaterialiseAlarmCondition(Free, ServerEventRig.RawDeviceFolder, "free", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true); + rig.Sink.WriteAlarmCondition(Free, Active("x1"), DateTime.UtcNow, AddressSpaceRealm.Raw); + + // EqA needs an event item of its own to be a notifier fan-out the stall can apply to. + using var eqASession = await rig.ConnectAsync(anonymous: false, ct); + var eqAEvents = new RecordedEvents(); + var eqASubscription = await RecordedEvents.SubscribeAsync(eqASession, eqAEvents, ct); + RecordedEvents.AddEventItem(eqASubscription, ServerEventRig.UnsNodeId(eqASession, EqA)); + await eqASubscription.ApplyChangesAsync(ct); + + // The refreshed item is on the raw device folder, which both conditions report through. + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onDevice = RecordedEvents.AddEventItem(subscription, RawNodeId(session, ServerEventRig.RawDeviceFolder)); + await subscription.ApplyChangesAsync(ct); + onDevice.Status.Error.ShouldBeNull(); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onDevice, "seed")); + + var injected = 0; + rig.Nm.DirectRefreshProbe = (cut, _) => + { + if (cut && Interlocked.Exchange(ref injected, 1) == 0) + rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw); + }; + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred"); + await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + await RecordedEvents.WaitUntilAsync(() => events.HasType(onDevice, ObjectTypeIds.RefreshEndEventType)); + stall.Release(); + } + + rig.Nm.DirectRefreshProbe = null; + Volatile.Read(ref injected).ShouldBe(1, "the folder refresh took the cut path"); + var stream = events.Of(onDevice).ToList(); + var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType); + var free = stream.Take(end).Select(RecordedEvents.Message).Where(m => m is "x1" or "x2").ToList(); + free.ShouldContain("x2"); + free[^1].ShouldBe("x2", "the older frozen replay of the condition must not follow its newer event"); + } + + /// + /// ACL on: a condition the folder refresh omitted because the item may not see it there (its reference into EqA was + /// dropped by the generation) is not withheld afterwards — the reference returns before the drain, and the item must + /// end on the condition's newest state. And while omitted, the replay itself carried nothing of it. + /// + [Fact] + public async Task Deferred_events_of_a_condition_the_folder_refresh_omitted_by_the_acl_are_not_withheld() + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Acl", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + NodeAcl[] grants = [KeyEquipmentGrant(EqA)]; + var state = rig.Enforce(true, [ServerEventRig.Key()], grants); + Deploy(state, "rev-2", grants, ServerEventRig.EqB); // the native alarm is no longer projected into EqA + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA)); + await subscription.ApplyChangesAsync(ct); + onA.Status.Error.ShouldBeNull(); + + rig.Project(native: true, Active("seed")); + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + events.Has(onA, "seed").ShouldBeFalse("the generation does not project the alarm into EqA"); + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + rig.Project(native: true, Active("final")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0); + + await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType)); + + // The reference returns while the condition's events still wait. + Deploy(state, "rev-3", grants, EqA, ServerEventRig.EqB); + stall.Release(); + } + + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "final")); + var (replayed, after) = Window(events.Of(onA)); + replayed.ShouldBeEmpty("the refresh omitted the condition: not visible through EqA at the time"); + after.ShouldNotBeEmpty(); + after[^1].ShouldBe("final", "the item ends on the newest state of a condition it was never refreshed"); + after.Count(m => m!.StartsWith('f')).ShouldBeGreaterThan(0, "its deferred events were delivered, not withheld"); + } + + /// + /// ACL on: a grant revoked while a refreshed folder item's deferred events wait. Neither the replay (taken while + /// granted) nor anything after it leaks once revoked; a still-granted subscriber on the same folder gets the tail. + /// + [Fact] + public async Task A_grant_revoked_after_a_folder_refresh_withholds_every_deferred_event() + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Revoke", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly); + var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant); + + using var keyed = await rig.ConnectAsync(anonymous: false, ct); + var keyEvents = new RecordedEvents(); + var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct); + var keyA = RecordedEvents.AddEventItem(keySub, ServerEventRig.UnsNodeId(keyed, EqA)); + await keySub.ApplyChangesAsync(ct); + + using var anonymous = await rig.ConnectAsync(anonymous: true, ct); + var anonEvents = new RecordedEvents(); + var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct); + var anonA = RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA)); + await anonSub.ApplyChangesAsync(ct); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(keyA, "seed") && anonEvents.Has(anonA, "seed")); + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id); + await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(keyA, ObjectTypeIds.RefreshEndEventType)); + + state.Apply( + ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(), + ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(), + ServerEventRig.TagReferences()); + rig.Project(native: true, Active("after-revoke")); + stall.Release(); + } + + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + await RecordedEvents.WaitUntilAsync(() => + anonEvents.Has(anonA, $"f{FloodSize - 1}") && anonEvents.Has(anonA, "after-revoke")); + await Task.Delay(TimeSpan.FromSeconds(1), ct); + + var (_, after) = Window(keyEvents.Of(keyA)); + after.ShouldBeEmpty("after RefreshEnd the revoked session gets nothing: older events are superseded, later ones denied"); + keyEvents.Has(keyA, "after-revoke").ShouldBeFalse(); + } + + /// + /// Production shape, no stall: bursts reported under one Lock hold (an apply pass) and single reports, against + /// folder items that refresh repeatedly while their client is slow to publish. Folder items are served per item, so + /// nothing is ever deferred; each refresh's replay is never older than an event the item already received, nothing + /// after RefreshEnd goes backwards, the stream converges on the newest state, and no replay is queued under Lock. + /// + /// Per-folder ACL on or off. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Production_overload_never_defers_and_never_overtakes_a_folder_refresh(bool enforced) + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Load", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA)); + onA.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream + await subscription.ApplyChangesAsync(ct); + + rig.Project(native: true, Active("s0")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0")); + + var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>(); + rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held)); + + using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct); + var counter = 0; + var peakDeferred = 0; + var reporter = Task.Factory.StartNew(() => + { + while (!stop.IsCancellationRequested) + { + lock (rig.Nm.Lock) + { + for (var i = 0; i < 1000; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); + } + + peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports); + rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); + Thread.Sleep(2); + } + }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + var refusedAsInProgress = 0; + for (var r = 0; r < 30; r++) + { + try + { + await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct); + } + catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress) + { + refusedAsInProgress++; + } + + await Task.Delay(TimeSpan.FromMilliseconds(80), ct); + } + + await stop.CancelAsync(); + await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None); + rig.Nm.DirectRefreshProbe = null; + + peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item by the reporter"); + rig.Nm.DiscardedEventReports.ShouldBe(0); + var newest = $"s{Volatile.Read(ref counter)}"; + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest)); + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + + CheckStream(events.Of(onA)).ShouldBeEmpty(); + refusedAsInProgress.ShouldBeLessThan(30, "most refreshes must actually have run"); + probes.ShouldNotBeEmpty(); + probes.ShouldAllBe(p => !p.Cut && !p.LockHeld); + RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))) + .ShouldBe(newest, "the stream converges on the newest state"); + } + + /// + /// Race / stress with deferral: reports (bursts under Lock and single), repeated refreshes of the folder item + /// and an intermittent stall of its folder's delivery all run at once. Outside a refresh the folder item's stream + /// never goes backwards, each refresh ends on the newest state the item has seen, and it converges. Nothing + /// deadlocks, and neither refresh path queues its replay under Lock. + /// + /// Per-folder ACL on or off. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Concurrent_reports_folder_refreshes_and_stalls_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced) + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Stress", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA)); + onA.QueueSize = 100_000; + await subscription.ApplyChangesAsync(ct); + + rig.Project(native: true, Active("s0")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0")); + + var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>(); + rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held)); + + using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct); + var counter = 0; + var reporter = Task.Factory.StartNew(() => + { + while (!stop.IsCancellationRequested) + { + lock (rig.Nm.Lock) + { + for (var i = 0; i < 300; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); + } + + rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); + Thread.Sleep(2); + } + }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + var deferredSeen = 0; + var staller = Task.Factory.StartNew(() => + { + while (!stop.IsCancellationRequested) + { + using (NotifierStall.Hold(rig.Nm, EqA)) + { + Thread.Sleep(300); + if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen); + } + + Thread.Sleep(100); + } + }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + var refusedAsInProgress = 0; + var stressDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(90); + for (var r = 0; r < 40 || (Volatile.Read(ref deferredSeen) <= 1 && DateTime.UtcNow < stressDeadline); r++) + { + try + { + await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct); + } + catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress) + { + refusedAsInProgress++; + } + + await Task.Delay(TimeSpan.FromMilliseconds(80), ct); + } + + await stop.CancelAsync(); + await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None); + await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None); + rig.Nm.DirectRefreshProbe = null; + Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events"); + + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + var newest = $"s{Volatile.Read(ref counter)}"; + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest)); + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + + CheckStream(events.Of(onA)).ShouldBeEmpty(); + refusedAsInProgress.ShouldBeLessThan(40, "most refreshes must actually have run"); + rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some folder refresh must actually have overtaken deferred events"); + probes.ShouldContain(p => p.Cut, "some folder refresh must have taken the cut path"); + probes.ShouldAllBe(p => !p.LockHeld); + RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))) + .ShouldBe(newest, "the stream converges on the newest state"); + } + + /// + /// Walks one condition's stream: outside a refresh its states never go backwards, and each refresh ends on the newest + /// state the item has seen (its replay is never older than an event already delivered, and nothing older follows). + /// + private static List CheckStream(IReadOnlyList> stream) + { + var max = -1; + var last = -1; + var replaying = false; + var violations = new List(); + foreach (var f in stream) + { + var type = f[RecordedEvents.EventTypeField].GetNodeId(); + if (type == ObjectTypeIds.RefreshStartEventType) + { + replaying = true; + continue; + } + + if (type == ObjectTypeIds.RefreshEndEventType) + { + replaying = false; + if (last < max) violations.Add($"a refresh ended on s{last} after s{max} had been delivered"); + continue; + } + + if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue; + var n = int.Parse(RecordedEvents.Message(f)![1..], CultureInfo.InvariantCulture); + if (!replaying && n < max) violations.Add($"s{n} delivered after s{max}"); + max = Math.Max(max, n); + last = n; + } + + return violations; + } + + private static Task RefreshAsync(ISession session, Subscription subscription, MonitoredItem item, bool refresh2, CancellationToken ct) => + refresh2 + ? session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, subscription.Id, item.Status.Id) + : session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + + /// The condition-event messages inside the item's LAST RefreshStart..RefreshEnd, and those after it. + private static (List Replayed, List AfterEnd) Window(IReadOnlyList> stream) + { + var list = stream.ToList(); + var start = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType); + var end = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType); + start.ShouldBeGreaterThanOrEqualTo(0); + end.ShouldBeGreaterThan(start); + static bool IsCondition(ArrayOf f) => f[RecordedEvents.AckedField].TryGetValue(out bool _); + return ( + list.Skip(start + 1).Take(end - start - 1).Where(IsCondition).Select(RecordedEvents.Message).ToList(), + list.Skip(end + 1).Where(IsCondition).Select(RecordedEvents.Message).ToList()); + } + + private static List Indices(IReadOnlyList> received, char prefix) => + received + .Select(RecordedEvents.Message) + .Where(m => m is { Length: > 1 } && m[0] == prefix && char.IsAsciiDigit(m[1])) + .Select(m => int.Parse(m![1..], CultureInfo.InvariantCulture)) + .ToList(); + + private static NodeId RawNodeId(ISession session, string id) => + new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri)); + + private static void Deploy(DataPlaneAuthorityState state, string revision, NodeAcl[] grants, params string[] referencing) => + state.Apply( + ServerEventRig.Cluster, revision, [ServerEventRig.Key()], grants, ServerEventRig.ScopePaths(), + ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(), + new Dictionary> { [ServerEventRig.RawAlarmPath] = referencing.ToHashSet() }); + + private static AlarmConditionSnapshot Active(string message) => + new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message); + + private static AlarmConditionSnapshot Cleared(string message) => + new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message); + + private static NodeAcl KeyEquipmentGrant(string equipment) => new() + { + NodeAclId = $"acl-key-{equipment}", + ClusterId = ServerEventRig.Cluster, + Principal = ServerEventRig.KeyPrincipal, + ScopeKind = NodeAclScopeKind.Equipment, + ScopeId = $"eq-{equipment}", + PermissionFlags = NodePermissions.ReadOnly, + }; +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs index d99ac83e6..97f2271de 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs @@ -19,6 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// consumer — a deadlock that also freezes every other operation needing Lock. Publishing is switched off /// on the client so nothing downstream drains either (the slow / absent consumer case). /// +/// +/// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a +/// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same +/// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in +/// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its +/// deferral with . +/// public sealed class FolderEventFloodWireTests { private const string EqA = ServerEventRig.EqA; @@ -56,8 +63,12 @@ public sealed class FolderEventFloodWireTests // No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client. await subscription.SetPublishingModeAsync(false, ct); + // #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself. + rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType(); + // The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection // pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1. + var peakDeferred = 0; var flood = Task.Factory.StartNew(() => { lock (rig.Nm.Lock) @@ -65,6 +76,7 @@ public sealed class FolderEventFloodWireTests for (var i = 0; i < FloodSize; i++) { rig.Project(native: true, State($"f{i}")); + peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports); Volatile.Write(ref reported, i + 1); } } @@ -83,11 +95,9 @@ public sealed class FolderEventFloodWireTests + "on Lock (deadlock)"); } - // The overflow was deferred rather than blocking, and drains once Lock is free. - await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); - // …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with - // publishing off, so the count below measures the item queue and not events in transit. - await Task.Delay(TimeSpan.FromSeconds(2), ct); + // #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself. + peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel"); + rig.Nm.DeferredEventReports.ShouldBe(0); // Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the // events that survive arrive in report order, ending with the last one reported. @@ -104,8 +114,7 @@ public sealed class FolderEventFloodWireTests flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood"); flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported"); flooded[^1].ShouldBe(FloodSize - 1); - rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, - "past the per-condition budget the oldest waiting events give way to the newest (coalesced)"); + rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)"); // And the node manager is live: a later event is decided by the per-folder ACL and delivered. rig.Project(native: true, State("after")); @@ -127,7 +136,8 @@ public sealed class FolderEventFloodWireTests /// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes /// the condition while its events wait used to let them reach Server-object (i=2253) subscribers after the /// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred - /// report now re-checks that the condition is still the live instance. + /// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before + /// the folder's items are served). /// /// Per-folder / Server-object ACL on or off (off is the production default). [Theory] @@ -151,16 +161,20 @@ public sealed class FolderEventFloodWireTests onServer.Status.Error.ShouldBeNull(); await subscription.SetPublishingModeAsync(false, ct); - // One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition. - await Task.Factory.StartNew(() => + // One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition. + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) { - lock (rig.Nm.Lock) + await Task.Factory.StartNew(() => { - for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}")); - rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events"); - rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue(); - } - }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct); + lock (rig.Nm.Lock) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events"); + rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue(); + } + }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct); + stall.Release(); + } await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered"); @@ -174,8 +188,8 @@ public sealed class FolderEventFloodWireTests .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture)) .ToArray(); - // Positive control: the events reported before the channel filled did arrive (Server object: synchronously at - // the condition's own hop). + // Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the + // condition's own hop, before the walk reached the stalled folder). Flooded(onServer).ShouldNotBeEmpty(); // None of the deferred tail — which was still waiting when the condition was removed — reached anyone. Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber"); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs index 37fe86526..f20a6ea7a 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs @@ -97,10 +97,10 @@ public sealed class FolderEventVerdictInvalidationTests rig.Project(native: false, State("s1")); await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s1")); - // Holding the node-manager lock parks EqA's SDK delivery task: the native event "n1" is queued first and its - // verdict (a cache miss after the first invalidation) needs the lock in the inherited role check — its node is - // never removed, so that check always takes it. The OLD scripted instance's "s2", queued behind it on the same - // single-reader queue, is therefore decided only after the replace and the apply's invalidation. + // Before Gitea #855, holding the node-manager lock parked EqA's SDK delivery task, so the OLD scripted instance's + // "s2" was decided (and its verdict cached) only after the replace and the apply's invalidation. Condition events + // are now decided per item at report time, uncached — "s2" is delivered while its instance is live — and this + // pins that the replace still leaves the new instance's events flowing either way. lock (rig.Nm.Lock) { rig.Nm.CompleteAddressSpaceApply(); // forget the cached allows, so both verdicts must be computed diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs new file mode 100644 index 000000000..755f26a09 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs @@ -0,0 +1,42 @@ +using Opc.Ua; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; + +/// +/// Stalls condition-event delivery at one of the node manager's notifiers (Gitea #855): every condition event whose walk +/// reaches that notifier suspends there until , without blocking the reporter or holding the node +/// manager's Lock — so the condition's later events are deferred, as they were behind a full SDK channel before +/// folder items were delivered per item. +/// +/// +/// The pre-#855 tests produced deferral by holding a folder item's server-side MonitoredItem lock, which parked +/// the SDK's per-node delivery task. Folder items are now queued synchronously by the reporter (as Server-object items +/// always were), so holding an item lock would block the reporter instead; this goes through the node manager's test +/// probe (). The probe is consulted only at a notifier that has +/// event items (the hop the per-item delivery serves), so the stalled notifier must carry at least one. +/// +internal sealed class NotifierStall : IDisposable +{ + private readonly OtOpcUaNodeManager _nm; + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly Func _probe; + + private NotifierStall(OtOpcUaNodeManager nm, string notifierId) + { + _nm = nm; + _probe = node => !_released.Task.IsCompleted && node.NodeId.IdentifierAsString == notifierId ? _released.Task : null; + nm.NotifierDeliveryStall = _probe; + } + + /// Starts stalling delivery at the notifier whose NodeId identifier is . + public static NotifierStall Hold(OtOpcUaNodeManager nm, string notifierId) => new(nm, notifierId); + + /// Lets every stalled walk continue (on pool threads, off Lock). + public void Release() => _released.TrySetResult(); + + public void Dispose() + { + Release(); + if (ReferenceEquals(_nm.NotifierDeliveryStall, _probe)) _nm.NotifierDeliveryStall = null; + } +} From 8e97fc94fd73112e8b3562a8a27fec7db77c580d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:20:33 -0500 Subject: [PATCH 08/28] fix(heidenhain): Tool/Number, Tool/Index, Tool/AxisCode publish their declared integer types (#867) HeidenhainSession.Leaf's `case Lsv2Tool t` switch expression mixed long, int and double? arms, so C# unified the natural type to double? and Tool/Number (Int64), Tool/Index and Tool/AxisCode (Int32) were published as Double. Every arm in that switch (and in the alarm-list, axes and feature-acquire switches in the same file) is now boxed to object explicitly so no numeric arm can widen another. New HeidenhainLeafRuntimeTypeTests asserts value.GetType() for every catalog value leaf against its declared DataType (scalar + array), plus PLC/DATA binding coercion for Int16/Int32/Int64/Boolean. Verified red on the old code (exactly the three Tool leaves) and green with the fix. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../HeidenhainSession.cs | 32 +++-- .../HeidenhainLeafRuntimeTypeTests.cs | 133 ++++++++++++++++++ 2 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests/HeidenhainLeafRuntimeTypeTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain/HeidenhainSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain/HeidenhainSession.cs index 0c2b831ed..e354caa35 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain/HeidenhainSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain/HeidenhainSession.cs @@ -256,14 +256,15 @@ internal sealed class HeidenhainSession : IAsyncDisposable await PrepareTelemetryAsync(wire, ct).ConfigureAwait(false); value = feature switch { - "Execution" => Lsv2Data.U16(await _client.ReadRuntimeAsync(23, ct).ConfigureAwait(false)), - "ProgramState" => Lsv2Data.U16(await _client.ReadRuntimeAsync(26, ct).ConfigureAwait(false)), + // Each arm boxed explicitly so no numeric arm can widen another (lmxopcua#867). + "Execution" => (object)Lsv2Data.U16(await _client.ReadRuntimeAsync(23, ct).ConfigureAwait(false)), + "ProgramState" => (object)Lsv2Data.U16(await _client.ReadRuntimeAsync(26, ct).ConfigureAwait(false)), "Program" => Lsv2Data.Program(await _client.ReadRuntimeAsync(24, ct).ConfigureAwait(false)), "Overrides" => Lsv2Data.Overrides(await _client.ReadRuntimeAsync(25, ct).ConfigureAwait(false)), "Axes" => Lsv2Data.Axes(await _client.ReadRuntimeAsync(22, ct).ConfigureAwait(false)), "Tool" => Lsv2Data.Tool(await _client.ReadRuntimeAsync(51, ct).ConfigureAwait(false)), HeidenhainCatalog.AlarmsFeature => await AlarmsAsync(ct).ConfigureAwait(false), - _ => await TimerAsync(feature, ct).ConfigureAwait(false), + _ => (object)await TimerAsync(feature, ct).ConfigureAwait(false), }; } @@ -444,24 +445,27 @@ internal sealed class HeidenhainSession : IAsyncDisposable case double[] o when address.StartsWith("Overrides/", StringComparison.Ordinal): return o[name switch { "Feed" => 0, "Spindle" => 1, _ => 2 }]; case Lsv2Axes a: - return name == "Names" ? a.Names : a.Positions; + return name == "Names" ? (object)a.Names : (object)a.Positions; case Lsv2Tool t: + // lmxopcua#867: every arm is cast to object. Without it C# unifies long/int/double? to double?, so + // Tool/Number (Int64) and Tool/Index + Tool/AxisCode (Int32) were published as Double. A null + // geometry still boxes to null (BadNotSupported upstream). return name switch { - "Number" => t.Number, - "Index" => t.Index, - "AxisCode" => t.Axis, - "Length" => t.Length, - _ => t.Radius, + "Number" => (object)t.Number, + "Index" => (object)t.Index, + "AxisCode" => (object)t.Axis, + "Length" => (object?)t.Length, + _ => (object?)t.Radius, }; case List list: return name switch { - "Count" => list.Count, - "Messages" => list.Select(x => x.Message).ToArray(), - "Classes" => list.Select(x => x.ErrorClass).ToArray(), - "Groups" => list.Select(x => x.Group).ToArray(), - _ => list.Select(x => x.Number).ToArray(), + "Count" => (object)list.Count, + "Messages" => (object)list.Select(x => x.Message).ToArray(), + "Classes" => (object)list.Select(x => x.ErrorClass).ToArray(), + "Groups" => (object)list.Select(x => x.Group).ToArray(), + _ => (object)list.Select(x => x.Number).ToArray(), }; case int i: return i; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests/HeidenhainLeafRuntimeTypeTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests/HeidenhainLeafRuntimeTypeTests.cs new file mode 100644 index 000000000..7c10bd2cb --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests/HeidenhainLeafRuntimeTypeTests.cs @@ -0,0 +1,133 @@ +using System.Buffers.Binary; +using System.Text; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests.Simulator; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests; + +/// +/// lmxopcua#867: the published value's RUNTIME type must be the CLR type of the leaf's declared data type. +/// Shouldly equality cannot catch this (7.0 equals 7L), so every assertion here compares GetType(). +/// +[Trait("Category", "Unit")] +public sealed class HeidenhainLeafRuntimeTypeTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static Type ClrType(DriverDataType type, bool isArray) + { + var scalar = type switch + { + DriverDataType.Boolean => typeof(bool), + DriverDataType.Int16 => typeof(short), + DriverDataType.Int32 => typeof(int), + DriverDataType.Int64 => typeof(long), + DriverDataType.Float64 => typeof(double), + DriverDataType.String => typeof(string), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + return isArray ? scalar.MakeArrayType() : scalar; + } + + /// Every fixed value leaf (capability strings excluded), with its declared type. + public static TheoryData CatalogValueLeaves() + { + var data = new TheoryData(); + foreach (var e in HeidenhainCatalog.Entries.Where(e => !e.IsCapability)) data.Add(e.Address); + return data; + } + + private static SimulatedControl FullyPopulatedControl() + { + var c = new SimulatedControl(); + c.Runtime[22] = () => SimReply.Ok("S_RI", [0, 1, .. Encoding.ASCII.GetBytes("1.5\0X\0")]); + c.Runtime[23] = () => SimReply.Ok("S_RI", [0, 4]); + c.Runtime[26] = () => SimReply.Ok("S_RI", [0, 7]); + c.Runtime[24] = () => SimReply.Ok("S_RI", [0, 0, 0, 5, .. "A\0B\0"u8.ToArray()]); + c.Runtime[25] = () => SimReply.Ok("S_RI", new byte[12]); + var alarm = new byte[10]; + BinaryPrimitives.WriteInt16BigEndian(alarm, 3); + BinaryPrimitives.WriteInt16BigEndian(alarm.AsSpan(2), 4); + BinaryPrimitives.WriteInt32BigEndian(alarm.AsSpan(4), 12); + alarm[8] = 65; + c.Runtime[27] = () => SimReply.Ok("S_RI", alarm); + c.Runtime[28] = () => SimReply.Refuse(57); + // 24-byte tool reply: number 49, index 1, axis 2, length 100.5, radius 3.25 (float64le). + var tool = new byte[24]; + BinaryPrimitives.WriteUInt32BigEndian(tool, 49); + BinaryPrimitives.WriteUInt16BigEndian(tool.AsSpan(4), 1); + BinaryPrimitives.WriteUInt16BigEndian(tool.AsSpan(6), 2); + BinaryPrimitives.WriteDoubleLittleEndian(tool.AsSpan(8), 100.5); + BinaryPrimitives.WriteDoubleLittleEndian(tool.AsSpan(16), 3.25); + c.Runtime[51] = () => SimReply.Ok("S_RI", tool); + for (var s = 30; s <= 34; s++) c.Runtime[s] = () => SimReply.Ok("S_RI", [0, 0, 1, 0]); + return c; + } + + [Theory] + [MemberData(nameof(CatalogValueLeaves))] + public async Task Catalog_leaf_publishes_the_clr_type_of_its_declared_data_type(string address) + { + HeidenhainCatalog.TryGet(address, out var entry).ShouldBeTrue(); + var control = FullyPopulatedControl(); + await using var sim = Lsv2Simulator.WithControl(control); + var client = new Lsv2.Lsv2Client("127.0.0.1", sim.Port, TimeSpan.FromSeconds(1)); + var tag = TestTags.Def(address); + var settings = new HeidenhainSessionSettings(true, true, false, 64, true, false, false); + await using var session = new HeidenhainSession(client, settings, (_, _) => ValueTask.FromResult("")); + await client.OpenAsync(Ct); + await session.InitializeAsync(Ct); + + var sample = (await session.ReadBatchAsync([tag], Ct))[0]; + + sample.Status.ShouldBe(HeidenhainStatus.Good, address); + sample.Value.ShouldNotBeNull(address); + sample.Value.GetType().ShouldBe(ClrType(entry!.DataType, entry.IsArray), address); + control.Violations.ShouldBeEmpty(); + } + + [Fact] + public void Every_integer_catalog_leaf_is_covered() + { + // Pins the theory's reach: the integer leaves #867 broke are in the data set. + var leaves = CatalogValueLeaves().Select(r => r.Data).ToList(); + leaves.ShouldContain("Tool/Number"); + leaves.ShouldContain("Tool/Index"); + leaves.ShouldContain("Tool/AxisCode"); + leaves.ShouldContain("Alarms/Count"); + leaves.ShouldContain("Program/Line"); + leaves.ShouldContain("Status/ExecutionState"); + leaves.Count(l => l.StartsWith("Timers/", StringComparison.Ordinal)).ShouldBe(HeidenhainCatalog.TimerNames.Count); + } + + [Theory] + [InlineData(DriverDataType.Int16, 42)] + [InlineData(DriverDataType.Int32, 42)] + [InlineData(DriverDataType.Int64, 42)] + [InlineData(DriverDataType.Int16, 42L)] + [InlineData(DriverDataType.Int32, 42L)] + [InlineData(DriverDataType.Int64, 42L)] + [InlineData(DriverDataType.Int16, 42.0)] + [InlineData(DriverDataType.Int32, 42.0)] + [InlineData(DriverDataType.Int64, 42.0)] + [InlineData(DriverDataType.Boolean, true)] + [InlineData(DriverDataType.Boolean, false)] + public void Binding_coercion_publishes_the_clr_type_of_the_declared_data_type(DriverDataType type, object native) + { + HeidenhainBindingCodec.Coerce(native, type).GetType().ShouldBe(ClrType(type, false)); + } + + [Theory] + [InlineData(HeidenhainPlcArea.Byte, new byte[] { 7 }, DriverDataType.Int16)] + [InlineData(HeidenhainPlcArea.Byte, new byte[] { 7 }, DriverDataType.Int32)] + [InlineData(HeidenhainPlcArea.Word, new byte[] { 7, 0 }, DriverDataType.Int64)] + [InlineData(HeidenhainPlcArea.Dword, new byte[] { 7, 0, 0, 0 }, DriverDataType.Int32)] + [InlineData(HeidenhainPlcArea.InputWord, new byte[] { 7, 0 }, DriverDataType.Int32)] + [InlineData(HeidenhainPlcArea.Marker, new byte[] { 1 }, DriverDataType.Boolean)] + public void Plc_binding_decode_publishes_the_clr_type_of_the_declared_data_type(HeidenhainPlcArea area, byte[] bytes, DriverDataType type) + { + HeidenhainBindingCodec.DecodePlc(area, bytes, type).GetType().ShouldBe(ClrType(type, false)); + } +} From c4bc1ef3096cd4c0293d2c0f9dbd068bbf5e01cc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:24:03 -0500 Subject: [PATCH 09/28] fix(galaxy): seed platforms with one template filter per request (#864) The gateway's shared GalaxyHierarchyProjector ANDs template_chain_contains entries, so the single request carrying both $WinPlatform and $AppEngine matched no object and the probe watcher always seeded 0 platforms. GatewayGalaxyPlatformSource now runs one paged sweep per template and unions/dedupes by tag name. The tests use a fake gateway that applies the projector's AND semantics (they fail on the old single request). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../Browse/GatewayGalaxyPlatformSource.cs | 69 +++++++----- .../GatewayGalaxyPlatformSourceTests.cs | 102 ++++++++++++++---- 2 files changed, 125 insertions(+), 46 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs index dfed16597..875bbddc9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs @@ -5,11 +5,19 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; /// /// Default : a platform-only DiscoverHierarchy sweep -/// over the gateway's — the request filters on -/// template_chain_contains ($WinPlatform, $AppEngine) and asks for no -/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the -/// address-space discovery pulls. Paged at the configured Repository.DiscoverPageSize. +/// over the gateway's — one sweep per platform template +/// ($WinPlatform, then $AppEngine), each filtered on that single +/// template_chain_contains token with no attributes, so it is a few rows per galaxy rather +/// than the full object + attribute dump the address-space discovery pulls. Paged at the +/// configured Repository.DiscoverPageSize. /// +/// +/// ⚠️ One template per request (#864). The gateway's shared +/// GalaxyHierarchyProjector.MatchesFilters ANDs the template_chain_contains entries — +/// an object must match every token — so a single request carrying both templates matches no +/// platform (a $WinPlatform's chain has no $AppEngine, and vice versa) and the probe +/// watcher seeded 0 platforms in production. The sweeps are unioned and de-duplicated by tag name. +/// public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource { /// Template-chain tokens that identify a platform / engine object. @@ -32,15 +40,16 @@ public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource => QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken); /// - /// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every - /// request carries the template filter and include_attributes = false; the reply is + /// The platform-only sweeps behind a delegate so it is unit-testable without a channel. One + /// paged sweep per entry of ; every request carries exactly that + /// one template token and include_attributes = false; the reply is /// re-filtered on the returned template chain (a gateway that ignores the filter still /// yields only platforms) and reduced to non-empty tag names. /// /// The raw DiscoverHierarchy RPC. /// Objects per page. /// Cancellation token for the walk. - /// Distinct platform tag names in gateway order. + /// Distinct platform tag names: $WinPlatform sweep first, then $AppEngine, each in gateway order. internal static async Task> QueryAsync( Func> discover, int pageSize, @@ -51,33 +60,37 @@ public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource var names = new List(); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - var seenPageTokens = new HashSet(StringComparer.Ordinal); - var pageToken = string.Empty; - do + foreach (var template in PlatformTemplates) { - var request = new DiscoverHierarchyRequest + var seenPageTokens = new HashSet(StringComparer.Ordinal); + var pageToken = string.Empty; + do { - PageSize = effective, - PageToken = pageToken, - IncludeAttributes = false, - }; - request.TemplateChainContains.Add(PlatformTemplates); + var request = new DiscoverHierarchyRequest + { + PageSize = effective, + PageToken = pageToken, + IncludeAttributes = false, + }; + // Exactly ONE token per request: the gateway ANDs template filters (#864). + request.TemplateChainContains.Add(template); - var reply = await discover(request, cancellationToken).ConfigureAwait(false); - foreach (var obj in reply.Objects) - { - if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue; - if (seen.Add(obj.TagName)) names.Add(obj.TagName); - } + var reply = await discover(request, cancellationToken).ConfigureAwait(false); + foreach (var obj in reply.Objects) + { + if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue; + if (seen.Add(obj.TagName)) names.Add(obj.TagName); + } - pageToken = reply.NextPageToken ?? string.Empty; - if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken)) - { - throw new InvalidOperationException( - $"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop."); + pageToken = reply.NextPageToken ?? string.Empty; + if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken)) + { + throw new InvalidOperationException( + $"Galaxy DiscoverHierarchy (platform query, {template}) returned a repeated page token '{pageToken}' — refusing to loop."); + } } + while (!string.IsNullOrWhiteSpace(pageToken)); } - while (!string.IsNullOrWhiteSpace(pageToken)); return names; } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs index a385ecfb8..130d98de3 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs @@ -15,33 +15,99 @@ public sealed class GatewayGalaxyPlatformSourceTests return o; } - /// Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered. - [Fact] - public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered() + /// + /// A fake gateway that applies the request filters with the SAME semantics as the shared + /// GalaxyHierarchyProjector.MatchesFilters: every template_chain_contains token must + /// match (AND, case-insensitive substring over the chain), category_ids is an OR set. Pages + /// the filtered result at the request's page size. + /// + private sealed class FilteringGateway(params GalaxyObject[] galaxy) { - var requests = new List(); - Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + public List Requests { get; } = []; + + public Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) { - requests.Add(req); + Requests.Add(req.Clone()); + var matched = galaxy.Where(o => Matches(o, req)).ToList(); + var offset = req.PageToken == "" ? 0 : int.Parse(req.PageToken.Split(':')[1]); var reply = new DiscoverHierarchyReply(); - if (req.PageToken == "") + reply.Objects.Add(matched.Skip(offset).Take(req.PageSize)); + if (offset + req.PageSize < matched.Count) { - reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]); - reply.NextPageToken = "p2"; - } - else - { - reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]); + reply.NextPageToken = $"{string.Join(",", req.TemplateChainContains)}:{offset + req.PageSize}"; } return Task.FromResult(reply); } - var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None); + private static bool Matches(GalaxyObject obj, DiscoverHierarchyRequest req) + { + if (req.CategoryIds.Count > 0 && !req.CategoryIds.Contains(obj.CategoryId)) return false; + foreach (var filter in req.TemplateChainContains) + { + if (!obj.TemplateChain.Any(t => t.Contains(filter, StringComparison.OrdinalIgnoreCase))) return false; + } + return true; + } + } - names.ShouldBe(["Plat01", "Eng01"]); - requests.Count.ShouldBe(2); - requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false); - requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine")); + /// + /// #864: against a gateway that ANDs template filters (as the real one does), every deployed + /// $WinPlatform AND every $AppEngine is found — the old single request carrying both + /// tokens matched none. Shape of the VD03 DEV galaxy: 3 platforms, 2 engines. + /// + [Fact] + public async Task Finds_every_platform_and_engine_against_an_AND_filtering_gateway() + { + var gw = new FilteringGateway( + Obj("CTRX_DEV", "$WinPlatform"), + Obj("DevWinPlatform", "$WinPlatform"), + Obj("JoePlatform", "$JoePlatformTemplate", "$WinPlatform"), + Obj("DevAppEngine", "$AppEngine"), + Obj("JoeAppEngine", "$AppEngine"), + Obj("Tank01", "$UserDefined"), + Obj("Area01", "$Area")); + + var names = await GatewayGalaxyPlatformSource.QueryAsync(gw.Discover, pageSize: 50, CancellationToken.None); + + names.ShouldBe(["CTRX_DEV", "DevWinPlatform", "JoePlatform", "DevAppEngine", "JoeAppEngine"]); + gw.Requests.ShouldAllBe(r => r.TemplateChainContains.Count == 1 && r.PageSize == 50 && r.IncludeAttributes == false); + gw.Requests.Select(r => r.TemplateChainContains[0]).Distinct().ShouldBe(["$WinPlatform", "$AppEngine"]); + } + + /// Each per-template sweep pages to completion independently; tag names are de-duplicated across sweeps. + [Fact] + public async Task Each_template_sweep_pages_independently_and_names_are_deduplicated() + { + var gw = new FilteringGateway( + Obj("Plat01", "$WinPlatform"), + Obj("Plat02", "$WinPlatform"), + Obj("Plat03", "$WinPlatform"), + // An object whose chain carries both templates is reported once. + Obj("Hybrid01", "$AppEngine", "$WinPlatform"), + Obj("Eng01", "$AppEngine"), + Obj("Eng02", "$AppEngine"), + Obj("", "$WinPlatform")); + + var names = await GatewayGalaxyPlatformSource.QueryAsync(gw.Discover, pageSize: 2, CancellationToken.None); + + names.ShouldBe(["Plat01", "Plat02", "Plat03", "Hybrid01", "Eng01", "Eng02"]); + // $WinPlatform: 5 matches (incl. the empty tag) => 3 pages; $AppEngine: 3 matches => 2 pages. + gw.Requests.Count(r => r.TemplateChainContains[0] == "$WinPlatform").ShouldBe(3); + gw.Requests.Count(r => r.TemplateChainContains[0] == "$AppEngine").ShouldBe(2); + } + + /// A gateway that ignores the filter still yields only platforms (replies are re-filtered on the chain). + [Fact] + public async Task Replies_are_refiltered_on_the_template_chain() + { + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + var reply = new DiscoverHierarchyReply(); + reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined"), Obj("Eng01", "$AppEngine")]); + return Task.FromResult(reply); + } + + (await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None)).ShouldBe(["Plat01", "Eng01"]); } /// A gateway that strips the template chain is trusted to have filtered server-side. From e35f2a4e93a605fa5963d1f886cc84b5359eef94 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:24:08 -0500 Subject: [PATCH 10/28] fix(galaxy): probe advises start the event stream first so initial ScanState arrives (#865) The gateway's StreamEvents with after_worker_sequence = 0 is a fresh stream and never replays, and the event pump started only on the first Read/Subscribe, so platform ScanState probes advised before then lost their initial value (and a driver with no authored tags never got any host status). PerPlatformProbeWatcher now awaits an ensure-event-stream hook before every probe SubscribeBulk (seed, discover sync, reconnect ResubscribeStaleAsync). The driver's hook starts the pump (refused while a teardown is in progress) and waits, bounded, on the new EventPump.StreamRequested signal - completed once the stream loop's first MoveNext has issued the StreamEvents RPC, or when the loop ends - so the advise never precedes the stream request. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 49 +++- .../Health/PerPlatformProbeWatcher.cs | 18 +- .../Runtime/EventPump.cs | 34 ++- .../Runtime/GalaxyDriverFixRound1Tests.cs | 5 +- .../Runtime/ProbeInitialValueTests.cs | 218 ++++++++++++++++++ 5 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index fd117b4f7..3102724f1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -540,6 +540,9 @@ public sealed class GalaxyDriver // a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It // runs in the background: InitializeAsync neither waits on it nor fails because of it. await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false); + // #865: the seed's probe advises start the event pump themselves (BuildProbeWatcher's + // pre-subscribe hook) — a driver with no authored tags never Reads/Subscribes, and the pump + // used to start only there, so every initial ScanState value went to no stream. RequestProbeSeed("initialize"); _logger.LogInformation( "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", @@ -730,9 +733,12 @@ public sealed class GalaxyDriver // The watcher publishes its probe handles into the driver's registry (reserved id) so the // EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every // probe event was dropped before it got anywhere near the watcher. + // #865: the watcher also starts the pump before every probe SubscribeBulk — the initial ScanState + // value only reaches a stream that is already attached (the gateway does not replay a fresh one). private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) => new(subscriber, _hostStatuses, _subscriptions, _logger, - bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs); + bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs, + ensureEventStream: EnsureProbeEventStreamAsync); /// /// Build the production gw client + session + per-capability runtime components @@ -2101,6 +2107,47 @@ public sealed class GalaxyDriver } } + /// + /// for infrastructure callers (the probe watcher) that + /// must not fail because the runtime is going away: false — and no pump — while a teardown is in + /// progress or there is no subscriber. + /// + /// The running (or just started) pump; null when none may be started. + private EventPump? TryEnsureEventPumpStarted() + { + lock (_pumpLock) + { + if (_runtimeStopping || _disposed || _subscriber is null) return null; + return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant); cannot throw here + } + } + + /// Upper bound on how long a probe subscribe waits for the pump's StreamEvents call to be issued. + internal TimeSpan ProbeStreamRequestWait { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// #865 — the probe watcher's pre-subscribe hook: start the pump (if this runtime may) and wait, + /// bounded, until its StreamEvents call has been issued, so the gateway has a subscriber to push + /// the probe's initial ScanState value to. A fresh stream (after_worker_sequence = 0) is never + /// replayed, so an initial value pushed before it attaches is lost — and ScanState rarely changes + /// again. On timeout the subscribe proceeds anyway (logged): a late stream is no worse than before. + /// + private async Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken) + { + var pump = TryEnsureEventPumpStarted(); + if (pump is null) return; + try + { + await pump.StreamRequested.WaitAsync(ProbeStreamRequestWait, cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + _logger.LogWarning( + "GalaxyDriver {InstanceId} event stream was not requested within {Wait} — subscribing the platform probes anyway; their initial ScanState may be missed.", + _driverInstanceId, ProbeStreamRequestWait); + } + } + /// /// Replay-gap callback for the . The gateway told us it evicted /// events we can never receive, so every cached value may be stale by an unknown amount. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs index fe887b084..08d2dd395 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs @@ -44,6 +44,11 @@ public sealed class PerPlatformProbeWatcher : IDisposable // driven directly (unit tests of the decoding table). private readonly SubscriptionRegistry? _registry; + // #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the + // gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events, + // and ScanState rarely changes — a lost initial value leaves the host missing indefinitely). + private readonly Func? _ensureEventStream; + // Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe; // we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call. // Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway @@ -79,16 +84,23 @@ public sealed class PerPlatformProbeWatcher : IDisposable /// The driver's subscription registry; null to keep the handles private. /// Optional logger for diagnostic messages. /// Buffered update interval in milliseconds; must be >= 0. + /// + /// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries + /// the initial ScanState value is running and requested (the driver starts its + /// and waits for its StreamEvents call to be issued). + /// internal PerPlatformProbeWatcher( IGalaxySubscriber subscriber, HostStatusAggregator aggregator, SubscriptionRegistry? registry, ILogger? logger = null, - int bufferedUpdateIntervalMs = 0) + int bufferedUpdateIntervalMs = 0, + Func? ensureEventStream = null) { _subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber)); _aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator)); _registry = registry; + _ensureEventStream = ensureEventStream; _logger = logger ?? NullLogger.Instance; if (bufferedUpdateIntervalMs < 0) { @@ -196,6 +208,10 @@ public sealed class PerPlatformProbeWatcher : IDisposable { var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray(); + // #865: the stream must be live BEFORE the advise — the initial value is the probe's only + // event until the platform's scan state changes, and an unattached stream never sees it. + if (_ensureEventStream is not null) await _ensureEventStream(cancellationToken).ConfigureAwait(false); + // #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until // PublishBindingsLocked registers the handles below (window closes at method exit). using var earlyEventWindow = _registry?.BeginPendingSubscribe(); diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index e2be7600e..c3640ab12 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -148,6 +148,17 @@ internal sealed class EventPump : IAsyncDisposable private readonly int _session; + private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Completes once the stream loop has issued its StreamEvents call (the first + /// MoveNextAsync runs the subscriber's iterator synchronously up to the RPC) — or once the + /// loop has ended without getting that far, so a waiter never hangs on a dead pump. #865: the + /// gateway does not replay a fresh stream, so a caller whose only expected event is the initial + /// value of an advise it is about to make waits on this first. + /// + public Task StreamRequested => _streamRequested.Task; + /// /// Marker written into the channel to wake the dispatch loop after a registration promoted /// parked early events (#859). Never dispatched itself; compared by reference. @@ -179,10 +190,17 @@ internal sealed class EventPump : IAsyncDisposable private async Task RunAsync(CancellationToken ct) { + IAsyncEnumerator? enumerator = null; try { - await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false)) + enumerator = _subscriber.StreamEventsAsync(ct).GetAsyncEnumerator(ct); + // Start the first MoveNext BEFORE awaiting it: that synchronous step issues the RPC, which is + // the moment StreamRequested reports (#865). + var next = enumerator.MoveNextAsync(); + _streamRequested.TrySetResult(); + for (; await next.ConfigureAwait(false); next = enumerator.MoveNextAsync()) { + var ev = enumerator.Current; if (ct.IsCancellationRequested) break; EventsReceived.Add(1, _clientTag); @@ -241,6 +259,20 @@ internal sealed class EventPump : IAsyncDisposable } finally { + // A loop that died before issuing the RPC must not strand a StreamRequested waiter. + _streamRequested.TrySetResult(); + if (enumerator is not null) + { + try { await enumerator.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) when (ct.IsCancellationRequested || ex is OperationCanceledException) + { + // Disposing a cancelled stream may surface its cancellation — shutdown noise. + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Galaxy EventPump stream enumerator dispose failed."); + } + } // Tell the dispatch loop the producer is done so it drains and exits. _channel.Writer.TryComplete(); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs index b22f8747c..2ca4698a7 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs @@ -303,6 +303,7 @@ public sealed class GalaxyDriverFixRound1Tests await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime"); + var oldPump = driver.EventPumpInstanceForTests; sub.ParkUnsubscribe = true; var shutdown = driver.ShutdownAsync(CancellationToken.None); @@ -330,7 +331,9 @@ public sealed class GalaxyDriverFixRound1Tests driver.TrackedSubscriptionCountForTests.ShouldBe(0); await driver.InitializeAsync("{}", CancellationToken.None); - driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump"); + // #865: the new runtime's background probe seed starts a pump of its own (its advises need a live + // stream) — but it must be a FRESH one, never the dead subscriber's. + driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump"); } // ───────────────────────────── IMPORTANT 3: bounded gateway close ───────────────────────────── diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs new file mode 100644 index 000000000..2144c8f7f --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs @@ -0,0 +1,218 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// #865: the platform ScanState probes' initial value. The gateway pushes an advise's current value +/// once, as its first event, and a FRESH StreamEvents (after_worker_sequence = 0) never +/// replays — so a probe advised before the driver's event stream is attached never gets a value, and +/// ScanState rarely changes again. The event pump used to start only on the first Read/Subscribe, +/// so a driver with no authored tags (or any driver before its first read) had no host statuses. +/// +public sealed class ProbeInitialValueTests +{ + private static readonly string[] Platforms = ["CTRX_DEV", "DevWinPlatform", "DevAppEngine"]; + + private static GalaxyDriverOptions Opts() => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("ProbeInitialValue"), + new GalaxyRepositoryOptions(WatchDeployEvents: false), + new GalaxyReconnectOptions()); + + private sealed class FixedPlatformSource(params string[] platforms) : IGalaxyPlatformSource + { + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => Task.FromResult>(platforms); + } + + /// + /// Models the gateway's event semantics: every advise pushes its current value exactly once, and + /// only to a stream that is attached at that moment (no replay for a fresh stream). Records the + /// order in which the stream was requested and the advises were made. + /// + private sealed class GatewayLikeSubscriber : IGalaxySubscriber + { + private readonly Lock _gate = new(); + private Channel? _attached; + private int _nextHandle; + + /// "stream" per StreamEvents call, "subscribe:<refs>" per SubscribeBulk, in call order. + public List Calls { get; } = []; + + /// Initial values pushed while no stream was attached — lost, as on the real gateway. + public int DroppedInitialValues; + + /// The ScanState value the next advise reports. + public volatile bool ScanStateRunning = true; + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var results = new List(); + lock (_gate) + { + Calls.Add("subscribe:" + string.Join(",", fullReferences)); + foreach (var r in fullReferences) + { + var h = ++_nextHandle; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + var initial = new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = h, + Value = new MxValue { BoolValue = ScanStateRunning }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }; + if (_attached is null || !_attached.Writer.TryWrite(initial)) Interlocked.Increment(ref DroppedInitialValues); + } + } + return Task.FromResult>(results); + } + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + // Attached synchronously on the first MoveNext — the point a real gRPC call is issued. + var mine = Channel.CreateUnbounded(); + lock (_gate) + { + Calls.Add("stream"); + _attached = mine; + } + try + { + await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + finally + { + lock (_gate) + { + if (ReferenceEquals(_attached, mine)) _attached = null; + } + } + } + + public string[] CallsSnapshot() + { + lock (_gate) return [.. Calls]; + } + } + + private static HostState? StateOf(GalaxyDriver driver, string host) => + driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; + + private static string Describe(GalaxyDriver driver, GatewayLikeSubscriber sub) => + $"hosts=[{string.Join(", ", driver.GetHostStatuses().Select(s => $"{s.HostName}={s.State}"))}] " + + $"calls=[{string.Join(" | ", sub.CallsSnapshot())}] dropped={sub.DroppedInitialValues} pump={driver.HasEventPumpForTests}"; + + /// + /// A driver with NO authored tags — nothing ever Reads or Subscribes — still reports every seeded + /// platform with its initial ScanState. Before #865 the pump never started, every initial value + /// was pushed to no stream, and only the transport entry ever appeared. + /// + [Fact] + public async Task Driver_with_no_authored_tags_reports_every_platform_from_its_initial_ScanState() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + platformSource: new FixedPlatformSource(Platforms)); + + await driver.InitializeAsync("{}", CancellationToken.None); + + await TestWait.UntilAsync( + () => Platforms.All(p => StateOf(driver, p) == HostState.Running), + "every seeded platform must surface Running from its initial ScanState, with no Read/Subscribe", + () => Describe(driver, sub)); + sub.DroppedInitialValues.ShouldBe(0, "no probe's initial value may be pushed before the stream is attached"); + driver.TrackedSubscriptionCountForTests.ShouldBe(1, "only the probe entry — no data subscription was ever made"); + } + + /// + /// Ordering: the event stream is requested BEFORE the first probe SubscribeBulk. On the old code + /// the probe advise went out with no stream at all. + /// + [Fact] + public async Task Probe_subscribe_is_issued_only_after_the_event_stream_is_requested() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + platformSource: new FixedPlatformSource(Platforms)); + + await driver.InitializeAsync("{}", CancellationToken.None); + await TestWait.UntilAsync( + () => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == Platforms.Length, + "positive control: the background seed must advise every platform", + () => Describe(driver, sub)); + + var calls = sub.CallsSnapshot(); + var firstSubscribe = Array.FindIndex(calls, c => c.StartsWith("subscribe:", StringComparison.Ordinal)); + var firstStream = Array.IndexOf(calls, "stream"); + firstSubscribe.ShouldBeGreaterThanOrEqualTo(0); + firstStream.ShouldBeGreaterThanOrEqualTo(0, $"the probe seed must start the event stream; calls=[{string.Join(" | ", calls)}]"); + firstStream.ShouldBeLessThan(firstSubscribe, $"the stream must be requested before the probe advise; calls=[{string.Join(" | ", calls)}]"); + } + + /// + /// A reconnect replay re-advises the probes on the recreated session; their new initial values must + /// arrive too (the replay restarts the pump first). The value flips to Stopped so the presence wait + /// can only pass on a post-replay event. + /// + [Fact] + public async Task Reconnect_replay_delivers_the_re_advised_probes_initial_values() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + platformSource: new FixedPlatformSource(Platforms)); + await driver.InitializeAsync("{}", CancellationToken.None); + await TestWait.UntilAsync( + () => Platforms.All(p => StateOf(driver, p) == HostState.Running), + "positive control: the initial seed reports Running", + () => Describe(driver, sub)); + + sub.ScanStateRunning = false; + await driver.InvokeReplayForTestAsync(CancellationToken.None); + + await TestWait.UntilAsync( + () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped), + "the re-advised probes' initial values (Stopped) must reach the host statuses after a replay", + () => Describe(driver, sub)); + sub.DroppedInitialValues.ShouldBe(0); + } + + /// + /// A probe sync on a driver whose pump was never started (no platform source, e.g. the + /// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just + /// the init path. + /// + [Fact] + public async Task Probe_sync_without_a_platform_source_still_starts_the_stream_first() + { + var sub = new GatewayLikeSubscriber(); + // No platform source: init starts no pump; the probe set arrives through SyncPlatformsAsync. + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeFalse("precondition: nothing has started the pump yet"); + + await driver.ProbeWatcherForTests.ShouldNotBeNull().SyncPlatformsAsync(Platforms, CancellationToken.None); + + await TestWait.UntilAsync( + () => Platforms.All(p => StateOf(driver, p) == HostState.Running), + "the watcher must start the stream before advising, so the initial values arrive", + () => Describe(driver, sub)); + sub.DroppedInitialValues.ShouldBe(0); + } +} From 02503a4a9f6092c6e4b08fd83f6656806b857e0b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:28:39 -0500 Subject: [PATCH 11/28] fix(controlplane): ConfigPublishCoordinator survives an unreachable ConfigDb at singleton start (#868) PreStart ran the in-flight recovery query synchronously. With the ConfigDb unreachable when the singleton started, the throw became an ActorInitializationException, the default strategy stopped the actor, the ClusterSingletonManager never re-created it, and every later deploy hung. - PreStart now only subscribes and kicks off an async recovery load piped back to the actor. A failure retries with exponential backoff (1 s doubling to 30 s, forever) and never stops the actor. - Until the load succeeds the coordinator is in a Loading behaviour that stashes every message (DispatchDeployment logged as queued, ApplyAck too); they are replayed in order after the recovered state is applied. Recovery decisions (re-dispatch / time out) now run inline before the replay instead of via self-tells, and a duplicate dispatch of the in-flight deployment with a seeded expected set is ignored so queued acks are never reset. - New ConfigPublishCoordinatorReadiness (admin-node DI singleton) + "config-publish" ready-tier health check: Degraded while loading (with the attempt count and last error), Healthy when ready or not hosted here. Tests: ConfigPublishCoordinatorDbOutageTests (DB down at start -> actor alive, Loading reported, deploy queued, DB returns -> load succeeds, queued deploy dispatched and sealed; queued ack applied to the recovered in-flight set; NotHosted after stop) and ConfigPublishCoordinatorHealthCheckTests. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../Coordinators/ConfigPublishCoordinator.cs | 225 ++++++++++++++++-- .../ConfigPublishCoordinatorReadiness.cs | 50 ++++ .../ServiceCollectionExtensions.cs | 6 +- .../ConfigPublishCoordinatorHealthCheck.cs | 45 ++++ .../Health/HealthEndpoints.cs | 9 + src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 5 + .../ConfigPublishCoordinatorDbOutageTests.cs | 185 ++++++++++++++ .../ConfigPublishCoordinatorRecoveryTests.cs | 6 +- ...onfigPublishCoordinatorHealthCheckTests.cs | 65 +++++ 9 files changed, 569 insertions(+), 27 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 01143dd4d..c96b682a9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -52,8 +52,20 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; /// ServerCluster.Enabled is not consulted — nothing else in the codebase consults it, /// so honouring it here would invent semantics. Disable the nodes, not the cluster row. /// +/// +/// Recovery load is asynchronous and retried (lmxopcua#868). The in-flight recovery read used to run +/// synchronously in PreStart. When the ConfigDb was unreachable as the singleton started, the throw +/// became an ActorInitializationException, the default strategy STOPPED the actor, and the +/// ClusterSingletonManager never re-creates a singleton that terminated on its own — so every later +/// deployment was told to a dead singleton and hung. The read now runs off the actor thread, a failure +/// retries with exponential backoff ( doubling to +/// , forever), and until it succeeds every message — deployments and acks +/// alike — is STASHED and replayed in order once the recovered state is in place, so a deploy arriving +/// during the outage is queued, never dropped. reports the +/// loading state (and the last error) to the node's health endpoint as Degraded. +/// /// -public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers +public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithUnboundedStash { public const string DeploymentsTopic = "deployments"; public const string DeploymentAcksTopic = "deployment-acks"; @@ -61,8 +73,18 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// apply-outcome timeout is derived from. public static readonly TimeSpan DefaultApplyDeadline = DeployTimings.ApplyDeadline; + /// First retry delay after a failed recovery load (lmxopcua#868). + public static readonly TimeSpan DefaultLoadRetryMin = TimeSpan.FromSeconds(1); + + /// Ceiling of the recovery-load retry backoff (lmxopcua#868). + public static readonly TimeSpan DefaultLoadRetryMax = TimeSpan.FromSeconds(30); + private readonly IDbContextFactory _dbFactory; private readonly TimeSpan _applyDeadline; + private readonly TimeSpan _loadRetryMin; + private readonly TimeSpan _loadRetryMax; + private readonly ConfigPublishCoordinatorReadiness? _readiness; + private int _failedLoads; /// /// Where central→node commands are handed off (per-cluster mesh Phase 2), or @@ -115,20 +137,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// Gets the timer scheduler for managing apply deadlines. public ITimerScheduler Timers { get; set; } = null!; + /// Holds every message that arrives before the recovery load completes (lmxopcua#868). + public IStash Stash { get; set; } = null!; + /// Creates actor props for the ConfigPublishCoordinator with the specified configuration. /// The database context factory for accessing configuration data. /// The timeout for waiting for apply acknowledgments from cluster nodes. - /// The Akka.NET used to spawn this actor. /// /// Central comm actor the dispatch is handed to, or to publish on the /// DistributedPubSub mediator directly (the original behaviour). /// + /// Where the recovery-load state is reported for health, or . + /// First recovery-load retry delay; = . + /// Recovery-load retry ceiling; = . + /// The Akka.NET used to spawn this actor. public static Props Props( IDbContextFactory dbFactory, TimeSpan? applyDeadline = null, - IActorRef? meshRouter = null) => - Akka.Actor.Props.Create(() => - new ConfigPublishCoordinator(dbFactory, applyDeadline ?? DefaultApplyDeadline, meshRouter)); + IActorRef? meshRouter = null, + ConfigPublishCoordinatorReadiness? readiness = null, + TimeSpan? loadRetryMin = null, + TimeSpan? loadRetryMax = null) => + Akka.Actor.Props.Create(() => new ConfigPublishCoordinator( + dbFactory, applyDeadline ?? DefaultApplyDeadline, meshRouter, readiness, + loadRetryMin ?? DefaultLoadRetryMin, loadRetryMax ?? DefaultLoadRetryMax)); /// /// Initializes a new instance of the class. @@ -136,45 +168,167 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// The database context factory for persisting deployment state. /// The timeout for waiting for per-node apply acknowledgments. /// Central comm actor, or for the mediator. + /// Where the recovery-load state is reported, or . + /// First recovery-load retry delay. + /// Recovery-load retry ceiling. public ConfigPublishCoordinator( IDbContextFactory dbFactory, TimeSpan applyDeadline, - IActorRef? meshRouter = null) + IActorRef? meshRouter = null, + ConfigPublishCoordinatorReadiness? readiness = null, + TimeSpan? loadRetryMin = null, + TimeSpan? loadRetryMax = null) { _dbFactory = dbFactory; _applyDeadline = applyDeadline; _meshRouter = meshRouter; + _readiness = readiness; + _loadRetryMin = loadRetryMin ?? DefaultLoadRetryMin; + _loadRetryMax = loadRetryMax ?? DefaultLoadRetryMax; + Loading(); + } + + /// + /// The state before the recovery load has succeeded: only the load's own outcome and retry tick are + /// handled; everything else is stashed and replayed, in arrival order, once is in force. + /// + private void Loading() + { + Receive(HandleRecoveryLoaded); + Receive(HandleRecoveryLoadFailed); + Receive(_ => StartRecoveryLoad()); + Receive(_ => { /* DPS subscribe confirmation */ }); + Receive(msg => + { + _log.Info( + "DispatchDeployment {Id} queued: the coordinator has not yet loaded its recovery state from the " + + "ConfigDb ({Failed} failed attempt(s) so far); it is dispatched as soon as the load succeeds", + msg.DeploymentId, _failedLoads); + Stash.Stash(); + }); + ReceiveAny(_ => Stash.Stash()); + } + + /// The normal operating state, entered once the recovery load has been applied. + private void Ready() + { Receive(HandleDispatch); Receive(HandleAck); Receive(HandleDeadline); Receive(_ => { /* DPS subscribe confirmation */ }); + // A late load outcome / retry tick cannot normally arrive once Ready; ignored, never stashed. + Receive(_ => { }); + Receive(_ => { }); + Receive(_ => { }); } /// protected override void PreStart() { - // On startup recover any deployment that was mid-flight when a prior singleton instance - // died. We re-derive _expectedAcks from NodeDeploymentState, replay the ACKs - // that already landed in the DB, and resume the deadline timer. // Subscribe to per-node ApplyAck broadcasts so DriverHostActors on remote members can // route their ACKs to whichever node currently hosts this singleton. DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self)); - using var db = _dbFactory.CreateDbContext(); + // lmxopcua#868: NO database access here. A throw from PreStart stops the actor for good (the + // singleton manager does not re-create it), so the recovery read runs asynchronously and retries. + _readiness?.MarkLoading(_failedLoads, lastError: null); + StartRecoveryLoad(); + } + + /// + protected override void PostStop() + { + // A restart re-enters PreStart (Loading again); a stop or hand-over leaves this node not hosting it. + _readiness?.MarkNotHosted(); + base.PostStop(); + } + + /// + /// Reads the deployment that was mid-flight when a prior singleton instance died, off the actor thread, + /// and pipes the outcome back. Touches no actor state. + /// + private void StartRecoveryLoad() + { + LoadRecoveryAsync(_dbFactory).PipeTo( + Self, + success: loaded => loaded, + failure: ex => new RecoveryLoadFailed(ex is AggregateException { InnerException: { } inner } ? inner : ex)); + } + + private static async Task LoadRecoveryAsync(IDbContextFactory dbFactory) + { + // Yield first so a factory or provider that throws synchronously still faults the TASK (piped back as + // RecoveryLoadFailed) rather than the caller on the actor thread. + await Task.Yield(); + await using var db = await dbFactory.CreateDbContextAsync().ConfigureAwait(false); // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never // reads — and this query runs on every singleton start and hand-over. - var inflight = db.Deployments + var inflight = await db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .OrderByDescending(d => d.CreatedAtUtc) .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc }) - .FirstOrDefault(); + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (inflight is null) return new RecoveryLoaded(null); + + var nodeStates = await db.NodeDeploymentStates + .Where(x => x.DeploymentId == inflight.DeploymentId) + .AsNoTracking() + .Select(x => new RecoveredNodeState(x.NodeId, x.Status, x.FailureReason)) + .ToListAsync() + .ConfigureAwait(false); + return new RecoveryLoaded(new RecoveredDeployment( + inflight.DeploymentId, inflight.RevisionHash, inflight.CreatedAtUtc, nodeStates)); + } + + private void HandleRecoveryLoadFailed(RecoveryLoadFailed msg) + { + _failedLoads++; + // Exponential backoff, capped: min * 2^(n-1), with the shift bounded so it never overflows. + var factor = 1L << Math.Min(_failedLoads - 1, 20); + var delay = TimeSpan.FromTicks(Math.Min(_loadRetryMax.Ticks, _loadRetryMin.Ticks * factor)); + _readiness?.MarkLoading(_failedLoads, msg.Error.Message); + _log.Warning(msg.Error, + "ConfigPublishCoordinator could not load its recovery state from the ConfigDb (attempt {Attempt}); " + + "retrying in {Delay}. Deployments and acks are queued until it succeeds", + _failedLoads, delay); + Timers.StartSingleTimer(LoadRetryTimerKey, RetryRecoveryLoad.Instance, delay); + } + + /// + /// Applies the recovered state, switches to and replays every stashed message. The + /// recovery DECISIONS (re-dispatch, time out) run inline, before the replay, so a queued message for the + /// same deployment is judged against the recovered state rather than racing a self-sent one. + /// + private void HandleRecoveryLoaded(RecoveryLoaded msg) + { + Timers.Cancel(LoadRetryTimerKey); + if (_failedLoads > 0) + _log.Info("ConfigPublishCoordinator loaded its recovery state after {Failed} failed attempt(s)", _failedLoads); + _failedLoads = 0; + Become(Ready); + _readiness?.MarkReady(); + try + { + ApplyRecovery(msg.Inflight); + } + finally + { + Stash.UnstashAll(); + } + } + + /// + /// Recovers the deployment that was mid-flight when a prior singleton instance died: re-derives + /// _expectedAcks from NodeDeploymentState, replays the acks that already landed, and resumes + /// the deadline timer. + /// + private void ApplyRecovery(RecoveredDeployment? inflight) + { if (inflight is null) return; _current = new DeploymentId(inflight.DeploymentId); - var nodeStates = db.NodeDeploymentStates - .Where(x => x.DeploymentId == inflight.DeploymentId) - .AsNoTracking() - .ToList(); + var nodeStates = inflight.NodeStates; // Resume the deadline timer using the remaining time. The deadline runs from when the // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough @@ -201,12 +355,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + "re-dispatching with {Remaining} of the apply deadline left", _current, remaining); - Self.Tell(new DispatchDeployment( + HandleDispatch(new DispatchDeployment( _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); } else { - Self.Tell(new DeadlineElapsed(_current.Value)); + HandleDeadline(new DeadlineElapsed(_current.Value)); } return; } @@ -222,18 +376,25 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers if (remaining <= TimeSpan.Zero) { - Self.Tell(new DeadlineElapsed(_current.Value)); - } - else - { - Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(_current.Value), remaining); + HandleDeadline(new DeadlineElapsed(_current.Value)); + return; } + + Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(_current.Value), remaining); _log.Info("Coordinator recovered in-flight deployment {Id} ({Acked}/{Total} acks landed)", _current, _acks.Count, _expectedAcks.Count); } private void HandleDispatch(DispatchDeployment msg) { + // A duplicate of the deployment already in flight with a seeded expected set (a dispatch queued while the + // recovery load ran, for the very row that load re-drove) must not reset the acks already gathered. + if (_current == msg.DeploymentId && _expectedAcks.Count > 0) + { + _log.Debug("Ignoring duplicate DispatchDeployment {Id}: already in flight", msg.DeploymentId); + return; + } + // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. @@ -519,9 +680,27 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers } private const string DeadlineTimerKey = "apply-deadline"; + private const string LoadRetryTimerKey = "recovery-load-retry"; public sealed record DeadlineElapsed(DeploymentId DeploymentId); + /// The recovery read's result: the newest in-flight deployment, or null when none. + private sealed record RecoveryLoaded(RecoveredDeployment? Inflight); + + /// The recovery read failed (ConfigDb unreachable, ...); retried with backoff. + private sealed record RecoveryLoadFailed(Exception Error); + + /// The backoff timer's tick: run the recovery read again. + private sealed class RetryRecoveryLoad + { + public static readonly RetryRecoveryLoad Instance = new(); + } + + private sealed record RecoveredDeployment( + Guid DeploymentId, string RevisionHash, DateTime CreatedAtUtc, IReadOnlyList NodeStates); + + private sealed record RecoveredNodeState(string NodeId, NodeDeploymentStatus Status, string? FailureReason); + /// /// Stages a status change WITHOUT materialising the row: the row carries the whole ArtifactBlob /// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs new file mode 100644 index 000000000..0074a8eee --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs @@ -0,0 +1,50 @@ +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; + +/// Where the singleton stands on this node (lmxopcua#868). +public enum ConfigPublishCoordinatorState +{ + /// The singleton is not running on this node (the partner hosts it, or it has not started yet). + NotHosted, + + /// Running here, but its recovery state has not loaded from the ConfigDb yet; deployments are queued. + Loading, + + /// Running here with its recovery state loaded; deployments are dispatched. + Ready, +} + +/// One immutable observation of the coordinator's state. +/// The state. +/// Consecutive failed recovery loads (0 unless ). +/// The last load failure's message, or null. +/// When the state was entered. +public sealed record ConfigPublishCoordinatorSnapshot( + ConfigPublishCoordinatorState State, int FailedLoadAttempts, string? LastError, DateTime SinceUtc); + +/// +/// Thread-safe state bridge from the actor to the node's health endpoint +/// (one DI singleton per admin node). The actor writes; the health check reads. +/// +public sealed class ConfigPublishCoordinatorReadiness +{ + private volatile ConfigPublishCoordinatorSnapshot _current = + new(ConfigPublishCoordinatorState.NotHosted, 0, null, DateTime.UtcNow); + + /// The latest observation. + public ConfigPublishCoordinatorSnapshot Current => _current; + + /// The coordinator is loading (or retrying the load of) its recovery state. + /// Failed attempts so far. + /// The last failure's message, or null. + public void MarkLoading(int failedAttempts, string? lastError) + { + var since = _current.State == ConfigPublishCoordinatorState.Loading ? _current.SinceUtc : DateTime.UtcNow; + _current = new(ConfigPublishCoordinatorState.Loading, failedAttempts, lastError, since); + } + + /// The recovery state is loaded. + public void MarkReady() => _current = new(ConfigPublishCoordinatorState.Ready, 0, null, DateTime.UtcNow); + + /// The singleton stopped on this node. + public void MarkNotHosted() => _current = new(ConfigPublishCoordinatorState.NotHosted, 0, null, DateTime.UtcNow); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index 9a9f07423..048613739 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -112,7 +112,11 @@ public static class ServiceCollectionExtensions // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks // inside a props factory and races the spawn it is waiting for. var meshRouter = registry.Get(); - return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter); + // lmxopcua#868: optional — through IServiceProvider for GetService's null-for-absent contract + // (the Akka resolver makes no such promise), so harnesses that never register it still boot. + var readiness = resolver.GetService()?.GetService(); + return ConfigPublishCoordinator.Props( + dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter, readiness: readiness); }, singletonOptions); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs new file mode 100644 index 000000000..579eb660d --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; + +namespace ZB.MOM.WW.OtOpcUa.Host.Health; + +/// +/// The deploy coordinator's readiness (lmxopcua#868): Degraded while the +/// singleton on this node has not yet loaded its recovery state from the +/// ConfigDb (deployments are queued, not dispatched), Healthy otherwise — including on the admin node that does +/// not host the singleton. Never Unhealthy: the node still serves (the configdb probe owns "DB down"). +/// +/// The coordinator's state bridge; null on graphs that never register one. +public sealed class ConfigPublishCoordinatorHealthCheck(ConfigPublishCoordinatorReadiness? readiness) : IHealthCheck +{ + /// The check's registration name. + public const string Name = "config-publish"; + + /// + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + if (readiness is null) + return Task.FromResult(HealthCheckResult.Healthy("Deploy coordinator readiness is not tracked on this node.")); + + var s = readiness.Current; + var data = new Dictionary(StringComparer.Ordinal) + { + ["state"] = s.State.ToString(), + ["sinceUtc"] = s.SinceUtc.ToString("O"), + ["failedLoadAttempts"] = s.FailedLoadAttempts, + }; + if (s.LastError is not null) data["lastError"] = s.LastError; + + return Task.FromResult(s.State switch + { + ConfigPublishCoordinatorState.Loading => HealthCheckResult.Degraded( + s.FailedLoadAttempts == 0 + ? "Deploy coordinator is loading its recovery state from the ConfigDb; deployments are queued until it completes." + : $"Deploy coordinator cannot load its recovery state from the ConfigDb ({s.FailedLoadAttempts} failed attempt(s), " + + $"last: {s.LastError}); retrying with backoff. Deployments are queued until it succeeds.", + data: data), + ConfigPublishCoordinatorState.Ready => HealthCheckResult.Healthy("Deploy coordinator ready.", data), + _ => HealthCheckResult.Healthy("Deploy coordinator singleton is not hosted on this node.", data), + }); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index e7fe87e91..24fec2e01 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -87,6 +87,15 @@ public static class HealthEndpoints { ProbeQuery = static (db, ct) => db.Deployments.AsNoTracking().Take(1).ToListAsync(ct), }); + + // lmxopcua#868: Degraded (still 200) while the deploy coordinator singleton on this node is retrying + // its ConfigDb recovery load and queueing deployments; Healthy otherwise. Ready tier only. + checks.Add(new HealthCheckRegistration( + ConfigPublishCoordinatorHealthCheck.Name, + sp => new ConfigPublishCoordinatorHealthCheck( + sp.GetService()), + failureStatus: null, + tags: new[] { ZbHealthTags.Ready })); } checks diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 132ead6f6..0d5d88ab6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -176,7 +176,12 @@ else builder.Services.Configure(builder.Configuration.GetSection(AclOptions.SectionName)); if (hasAdmin) +{ builder.Services.AddOtOpcUaConfigDb(builder.Configuration); + // lmxopcua#868: the deploy coordinator singleton reports its ConfigDb recovery-load state here; the + // config-publish health check (AddOtOpcUaHealth) reads it — Degraded while deployments are queued. + builder.Services.AddSingleton(); +} else Log.Information("Driver-only node — no ConfigDb registered; configuration via ConfigSource:Mode=FetchAndCache"); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs new file mode 100644 index 000000000..0e702382a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs @@ -0,0 +1,185 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// lmxopcua#868: the coordinator's recovery read used to run synchronously in PreStart. With the +/// ConfigDb unreachable when the singleton started, the actor died on initialization, was never re-created, +/// and every deployment hung. It now loads asynchronously with backoff, queues what arrives meanwhile and +/// reports until the load succeeds. +/// +public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTestBase +{ + private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('d', 64)); + private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan RetryMin = TimeSpan.FromMilliseconds(50); + private static readonly TimeSpan RetryMax = TimeSpan.FromMilliseconds(200); + private const string Node = "site-a-1:4053"; + + [Fact] + public void Db_unreachable_at_start_does_not_kill_the_coordinator_and_a_deploy_sent_during_the_outage_completes_once_it_returns() + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true }; + SeedNodes(direct, Node); + var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow); + var readiness = new ConfigPublishCoordinatorReadiness(); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + Watch(coordinator); + + // Several failed loads: the actor keeps retrying (it is alive) and reports Loading with the error. + AwaitAssert(() => + { + outage.Attempts.ShouldBeGreaterThanOrEqualTo(3); + readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Loading); + readiness.Current.FailedLoadAttempts.ShouldBeGreaterThanOrEqualTo(2); + readiness.Current.LastError.ShouldNotBeNull(); + readiness.Current.LastError.ShouldContain("unreachable"); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + // A deployment dispatched during the outage is queued, not processed and not dropped. + coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); + ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + using (var db = direct.CreateDbContext()) + { + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); + db.NodeDeploymentStates.ShouldBeEmpty(); + } + + // The DB comes back: the load succeeds, the queued dispatch runs, and the node's ack seals it. + outage.Down = false; + AwaitAssert(() => + { + readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready); + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node }); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + ExpectNoMsg(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); // never Terminated + } + + [Fact] + public void In_flight_deployment_is_recovered_after_the_outage_and_an_ack_queued_during_it_is_not_lost() + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true }; + SeedNodes(direct, Node, "site-a-2:4053"); + var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); + using (var db = direct.CreateDbContext()) + { + db.NodeDeploymentStates.Add(new NodeDeploymentState + { NodeId = Node, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying }); + db.NodeDeploymentStates.Add(new NodeDeploymentState + { NodeId = "site-a-2:4053", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, AppliedAtUtc = DateTime.UtcNow }); + db.SaveChanges(); + } + + var readiness = new ConfigPublishCoordinatorReadiness(); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2), + duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + // The last outstanding ack arrives while the coordinator is still loading: it is queued, then applied + // against the recovered expected set (site-a-2 already Applied in the DB) and seals the deployment. + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + outage.Down = false; + + AwaitAssert(() => + { + readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready); + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + [Fact] + public void Readiness_is_NotHosted_after_the_coordinator_stops() + { + var readiness = new ConfigPublishCoordinatorReadiness(); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory(), NoTimeout, readiness: readiness)); + AwaitAssert(() => readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready), + duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + Watch(coordinator); + Sys.Stop(coordinator); + ExpectTerminated(coordinator, cancellationToken: TestContext.Current.CancellationToken); + readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.NotHosted); + } + + private static DeploymentId SeedDeployment( + IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) + { + var id = DeploymentId.NewId(); + using var db = dbFactory.CreateDbContext(); + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, + RevisionHash = TestRevision.Value, + Status = status, + CreatedBy = "test", + CreatedAtUtc = createdAtUtc, + }); + db.SaveChanges(); + return id; + } + + private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) + { + using var db = dbFactory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + foreach (var nodeId in nodeIds) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], + ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", + }); + } + db.SaveChanges(); + } + + /// A factory that throws while — a ConfigDb that is unreachable. + private sealed class OutageDbFactory(IDbContextFactory inner) : IDbContextFactory + { + private volatile bool _down; + private int _attempts; + + public bool Down { get => _down; set => _down = value; } + + public int Attempts => Volatile.Read(ref _attempts); + + public OtOpcUaConfigDbContext CreateDbContext() + { + Interlocked.Increment(ref _attempts); + if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated outage)"); + return inner.CreateDbContext(); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index bd73313ad..91a947c8d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -136,9 +136,9 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); - // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and - // the actor would be stopped, not restarted). Identify is processed after creation, so a - // resolved selection proves PreStart completed. + // Arm the fault AFTER PreStart has run. Since lmxopcua#868 the recovery read is asynchronous and + // retried, so a fault that lands on it instead of on HandleDispatch is also recovered (the dispatch is + // queued until the retried load succeeds) — either way the deployment must complete. await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget, TestContext.Current.CancellationToken); faulting.FailNext(); admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs new file mode 100644 index 000000000..a9202432c --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs @@ -0,0 +1,65 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Health; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.Host.Health; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health; + +/// lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb. +public sealed class ConfigPublishCoordinatorHealthCheckTests +{ + private static async Task Check(ConfigPublishCoordinatorReadiness? readiness) => + await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None); + + [Fact] + public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error() + { + var r = new ConfigPublishCoordinatorReadiness(); + r.MarkLoading(3, "connection refused"); + var result = await Check(r); + result.Status.ShouldBe(HealthStatus.Degraded); + result.Description.ShouldNotBeNull(); + result.Description.ShouldContain("connection refused"); + result.Description.ShouldContain("queued"); + } + + [Fact] + public async Task Ready_is_healthy() + { + var r = new ConfigPublishCoordinatorReadiness(); + r.MarkLoading(2, "x"); + r.MarkReady(); + (await Check(r)).Status.ShouldBe(HealthStatus.Healthy); + } + + [Fact] + public async Task Not_hosted_and_untracked_are_healthy() + { + (await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy); + (await Check(null)).Status.ShouldBe(HealthStatus.Healthy); + } + + [Theory] + [InlineData(true, true)] + [InlineData(false, false)] + public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered) + { + var registrations = new ServiceCollection() + .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true) + .BuildServiceProvider() + .GetRequiredService>() + .Value.Registrations + .Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name) + .ToList(); + if (!registered) + { + registrations.ShouldBeEmpty(); + return; + } + registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]); + } +} From 8a0032bfdfe8d23057188db13bc5657713d38fb1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:30:01 -0500 Subject: [PATCH 12/28] fix(aggregates): live TimeAverage/Total leave the span before an Uncertain first value out (#821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the data starts inside an interval (no start bound) and its first value is Uncertain, the frozen 1.5.378.182 AverageAggregateCalculator integrated the uncovered span as a ramp from 0 and divided TimeAverage by the whole interval: GetRegionsInValueSet (ignoreBadData, sloped) overwrites the BadNoData start region's status with UncertainDataSubNormal in its one branch without a not-Bad guard. Live AggregateFilter therefore disagreed with HistoryRead and Part 13 §5.4.3.6/§5.4.3.8 ("No Start Bound: Calculate Partial Interval"); QP: Total 188.96 vs 163.96, TimeAverage 9.448 vs 10.931. SDK 2.0.1 stock does the same. Owner-approved 2026-11-11. - New app-owned CoveredTimeAverageAggregateCalculator subclasses the baseline and recomputes only TimeAverage/Total, restoring the Bad status of a region that starts at a Bad point (only the start bound can). Only such an interval's value changes; status, timestamps, type and bits are the baseline's. Stepped and TreatUncertainAsBad=true never reached the defect. CompatibilityAggregate- Factory routes exactly those two IDs; copied SDK files untouched. - Golden test: 0 fixture vectors change (every data set has a usable value at -10 s), asserted. New sweep over late-start variants of every data set (cut +5 s / +25 s, first non-Bad sample Uncertain, all directions/stepping/configs/ incremental): 150 / 142 of 384 change, each pinned to a baseline-derived oracle (untouched baseline's value with that sample Good, its status/bits on the variant itself) - never our calculator. Plus a hand-computed case. - Live-vs-HistoryRead parity (TimeWeightedAggregator) forward + reverse, no masks, with discrimination against the pre-#821 baseline; QualityMix now compares QP's TimeAverage/Total with the live oracle on both SQL backends; booted-runtime facts test expects 163.96 / 10.93. - README recorded decision, ServerAggregateConfiguration remarks, oracle remarks, docs/Historian.md and .claude/rules/historian.md updated. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .claude/rules/historian.md | 2 +- docs/Historian.md | 23 +-- .../CompatibilityAggregateFactory.cs | 10 +- .../CoveredTimeAverageAggregateCalculator.cs | 115 +++++++++++++++ .../Compatibility/Aggregates/README.md | 9 ++ .../ServerAggregateConfiguration.cs | 8 +- .../QualityMixConformance.cs | 29 ++-- .../SdkAggregateOracle.cs | 21 ++- .../SdkAggregateParityTests.cs | 80 +++++++++-- .../LiveAggregateLateStartFactsTests.cs | 18 +-- .../Sdk2AggregateCompatibilityTests.cs | 136 +++++++++++++++++- 11 files changed, 398 insertions(+), 53 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CoveredTimeAverageAggregateCalculator.cs diff --git a/.claude/rules/historian.md b/.claude/rules/historian.md index 7c8bf0050..507a2430e 100644 --- a/.claude/rules/historian.md +++ b/.claude/rules/historian.md @@ -32,7 +32,7 @@ historian tagname, which is the tag's driver `FullName`. ### Pluggable backends, SQL Server and HistoryUpdate (2026-10) - **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`. -- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; since #809 the live AggregateFilter agrees — `GoodOnlyStatusAggregateCalculator` overrides only those two of the frozen calculators) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. +- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; since #809 the live AggregateFilter agrees — `GoodOnlyStatusAggregateCalculator` overrides only those two of the frozen calculators; since #821 live TimeAverage/Total also leave the span before an Uncertain first sample out when the data starts inside an interval (no start bound — Part 13 partial interval), via `CoveredTimeAverageAggregateCalculator`) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates. - ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads". diff --git a/docs/Historian.md b/docs/Historian.md index e160a56b1..6d394d7ab 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -1906,16 +1906,19 @@ timestamp, and the same treatment as the Min/Max-at-interval-start case above; m And an interval holding a **Good String (or DateTime) sample** is `BadAggregateInvalidInputs` for Average/Minimum/Maximum/Range/ *ActualTime here (the documented #684 refusal rule — a Good non-numeric value is not a number to aggregate), while the SDK skips the string and serves the numeric samples (e.g. Average 4 Good); an Uncertain non-numeric sample refuses nothing on either side. -One more, found by the #810 Uncertain conformance data (kept deliberately): over the interval the data **starts inside** when the -**first sample is Uncertain**, the SDK's `TimeAverage`/`Total` integrate the uncovered span before it as a ramp from 0 (QP: `Total` -188.96 for 163.96 over the covered 15 s, and `TimeAverage` 9.448 = 188.96 ÷ 20 s for 10.931 = 163.96 ÷ 15 s); ours integrates only -the covered part (Part 13, as with a Good first sample, where all agree). The same answer comes from the frozen calculators the server -registers for **live** `AggregateFilter` items and from SDK 2.0.1's own stock calculators, in both read directions — pinned by -`SdkAggregateParityTests.An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data` -(ours / frozen / stock 2.0.1, with a Good-first control) and `OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests` (through -the booted server's runtime `AggregateManager`), and on both SQL backends by `QualityMixConformance.Time_weighted_values_over_partial_data_cover_only_the_data` -(forward) and `…_in_a_reverse_read` (#821). Whether live subscriptions should follow is the open #821 owner decision (#809, the -PercentGood/DurationGood half, is decided: live follows Part 13). The same +One more, found by the #810 Uncertain conformance data: over the interval the data **starts inside** when the **first sample is +Uncertain**, the SDK's `TimeAverage`/`Total` integrate the uncovered span before it as a ramp from 0 (QP: `Total` 188.96 for 163.96 +over the covered 15 s, and `TimeAverage` 9.448 = 188.96 ÷ 20 s for 10.931 = 163.96 ÷ 15 s); ours integrates only the covered part +(Part 13 "No Start Bound: Calculate Partial Interval", as with a Good first sample, where all agree). SDK 2.0.1's stock calculators +answer the same as the frozen 1.5.378.182 ones, in both read directions. ⚠️ **Since #821 (owner-approved 2026-11-11, client-visible) +the LIVE `AggregateFilter` calculators follow Part 13 here too** — `CoveredTimeAverageAggregateCalculator` overrides only +TimeAverage/Total of the frozen copies (only the value of such an interval changes; status, timestamps and bits are the baseline's). +Pinned by `SdkAggregateParityTests.An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data` +(ours / live / untouched baseline / stock 2.0.1, with a Good-first control), `…Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start` +(forward + reverse, no masks), `OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests` (through the booted server's runtime +`AggregateManager`), `Sdk2AggregateCompatibilityTests.Issue821_*`, and on both SQL backends by `QualityMixConformance` (QP's +TimeAverage/Total against the live oracle, and `Time_weighted_values_over_partial_data_cover_only_the_data` forward / `…_in_a_reverse_read`). +(#809, the PercentGood/DurationGood half, is likewise decided: live follows Part 13.) The same suite feeds PercentGood/PercentBad, DurationGood/DurationBad (and Count, Average, TimeAverage(2), Total(2), Start/End/Delta, Interpolative) Good/Uncertain/Bad mixes, all-Uncertain intervals (0 % Good **and** 0 % Bad, Mantis 0006166) and partial-data intervals, hand-computed and against the #796 oracle — the baseline calculator's Uncertain-as-Good PercentGood fails it. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs index 3be72a299..b6a13a9c9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CompatibilityAggregateFactory.cs @@ -8,14 +8,18 @@ internal static class CompatibilityAggregateFactory { /// /// PercentGood / DurationGood are served by the application-owned - /// (#809: Uncertain time is not Good, as in Part 13 and HistoryRead); every other id is the unchanged baseline. + /// (#809: Uncertain time is not Good, as in Part 13 and HistoryRead), TimeAverage / Total by + /// (#821: no start bound ⇒ only the covered part of the interval, + /// even when the first value is Uncertain); every other id is the unchanged baseline. /// internal static IAggregateCalculator Create(NodeId id, DateTimeUtc start, DateTimeUtc end, double interval, bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry) => new Adapter(GoodOnlyStatusAggregateCalculator.Overrides(id) ? new GoodOnlyStatusAggregateCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) - : Aggregators.CreateStandardCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) - ?? throw new ArgumentOutOfRangeException(nameof(id), "Unsupported baseline aggregate.")); + : CoveredTimeAverageAggregateCalculator.Overrides(id) + ? new CoveredTimeAverageAggregateCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) + : Aggregators.CreateStandardCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry) + ?? throw new ArgumentOutOfRangeException(nameof(id), "Unsupported baseline aggregate.")); /// Replaces exactly the existing standard factories; no aggregate IDs are added or removed. internal static async ValueTask RegisterAsync(AggregateManager manager, CancellationToken ct) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CoveredTimeAverageAggregateCalculator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CoveredTimeAverageAggregateCalculator.cs new file mode 100644 index 000000000..73f50c6ea --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/CoveredTimeAverageAggregateCalculator.cs @@ -0,0 +1,115 @@ +using Opc.Ua; +using DataValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.LegacyDataValue; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates; + +/// +/// Application-owned override (#821) of the baseline for TimeAverage +/// and Total only: the time before the first usable value (no start bound) is never integrated, whatever the +/// quality of that first value. Part 13 §5.4.3.6/§5.4.3.8 "No Start Bound: Calculate Partial Interval" — the rule the +/// HistoryRead evaluator (TimeWeightedAggregator, SqlServer + QuestDB) already follows, and the one the baseline +/// itself follows when the first value is Good. +/// +/// +/// +/// With no usable value before the interval, the baseline's interpolated start bound is BadNoData (value 0), +/// so the first region [start, first value) is Bad and excluded. But GetRegionsInValueSet +/// (ignoreBadData, sloped) marks the region before an Uncertain value UncertainDataSubNormal without the +/// not-Bad guard every other branch has — so an Uncertain first value turned the Bad region non-Bad and the +/// uncovered span was integrated as a ramp from 0 (QP: Total 188.96 for Part 13's 163.96, TimeAverage 9.448 for +/// 10.931). SDK 2.0.1's stock calculator does the same. +/// +/// +/// This class leaves the copied file untouched and restores exactly that one overwritten status: a region that +/// starts at a Bad point (only the interpolated start bound can — the region machinery skips Bad raw values) +/// keeps that point's Bad status. Every other region, the values, the result status (Uncertain either way: the +/// region is non-Good), timestamp, value type and Calculated / Partial bits are the baseline's. +/// TimeAverage2 / Total2 (simple bounds, no ignoreBadData) and stepped variables never reach that +/// branch and fall through to the baseline unchanged. +/// +/// +internal sealed class CoveredTimeAverageAggregateCalculator : AverageAggregateCalculator +{ + /// True for the two aggregate ids this class overrides. + internal static bool Overrides(NodeId aggregateId) => + aggregateId == ObjectIds.AggregateFunction_TimeAverage || aggregateId == ObjectIds.AggregateFunction_Total; + + /// Initializes the calculator; same contract as the baseline constructor. + public CoveredTimeAverageAggregateCalculator( + NodeId aggregateId, + DateTime startTime, + DateTime endTime, + double processingInterval, + bool stepped, + AggregateConfiguration configuration, + ITelemetryContext? telemetry) + : base(aggregateId, startTime, endTime, processingInterval, stepped, configuration, telemetry) + { + } + + /// + protected override DataValue ComputeValue(TimeSlice slice) + { + if (AggregateId == ObjectIds.AggregateFunction_TimeAverage) return ComputeCovered(slice, total: false); + if (AggregateId == ObjectIds.AggregateFunction_Total) return ComputeCovered(slice, total: true); + return base.ComputeValue(slice); + } + + /// + /// The baseline ComputeTimeAverage(slice, useSimpleBounds: false, valueType) with a region that starts at a + /// Bad point kept Bad. + /// + private DataValue ComputeCovered(TimeSlice slice, bool total) + { + List? values = GetValuesWithInterpolatedBounds(slice); + if (values == null || values.Count == 0) + { + return GetNoDataValue(slice); + } + + List regions = GetRegionsInValueSet(values, true, Stepped); + + double area = 0; + double totalDuration = 0; + bool nonGoodRegionsExists = false; + + foreach (var region in regions) + { + // #821: the one change — undo the baseline's Bad → Uncertain overwrite of a region starting at a Bad point. + if (StatusCode.IsBad(region.DataPoint.StatusCode) && StatusCode.IsNotBad(region.StatusCode)) + { + region.StatusCode = region.DataPoint.StatusCode; + } + + double duration = region.Duration / 1000.0; + + if (StatusCode.IsNotBad(region.StatusCode)) + { + area += (region.StartValue + region.EndValue) * duration / 2; + totalDuration += duration; + } + + if (StatusCode.IsNotGood(region.StatusCode)) + { + nonGoodRegionsExists = true; + } + } + + if (totalDuration == 0) + { + return GetNoDataValue(slice); + } + + double result = total ? area : area / totalDuration; + + var value = new DataValue + { + WrappedValue = SdkValueBoundary.FromClr(result), + SourceTimestamp = GetTimestamp(slice), + ServerTimestamp = GetTimestamp(slice), + StatusCode = nonGoodRegionsExists ? StatusCodes.UncertainDataSubNormal : StatusCodes.Good + }; + value.StatusCode = value.StatusCode.WithAggregateBits(AggregateBits.Calculated); + return value; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md index 1ba0f0c95..e5848b9fb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Compatibility/Aggregates/README.md @@ -28,3 +28,12 @@ The baseline `StatusAggregateCalculator.ComputeDurationGoodBad` counts Uncertain - `GoodOnlyStatusAggregateCalculator.cs` (application code, not a baseline copy) subclasses the baseline `StatusAggregateCalculator` and replaces only the PercentGood / DurationGood computation: the same slice values, simple bounds, stepped regions, empty-slice `BadNoData`, timestamps, value type and Calculated bit, with a region counted as Good only when its status is Good — whatever `TreatUncertainAsBad` says (when true, Uncertain is Bad, so it was never Good). `CompatibilityAggregateFactory.Create` routes exactly those two IDs to it; the other 35 IDs, including DurationBad / PercentBad / WorstQuality, are the baseline unchanged. - Golden test: the 64 vectors this changes (38 PercentGood + 26 DurationGood, all with an effective `TreatUncertainAsBad = false` over data with Uncertain regions) are exempted **exactly** — the PercentGood / DurationGood vectors whose baseline output differs from the same case's baseline vector with `TreatUncertainAsBad = true`, where 1.5.378.182 already counted Good regions only — and each is pinned to that twin's output. The pin is therefore baseline evidence, never regenerated from the application's calculator; any further drift in any of the 6,144 cases fails. `Sdk2AggregateCompatibilityTests.Issue809_*` also pins a hand-computed case. - Live-vs-history parity: `SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples` compares the registered calculator with `TimeWeightedAggregator` over Good / Uncertain / Bad mixes with no masks, and checks that the data discriminates against the pre-#809 baseline. + +### #821 — TimeAverage / Total integrate only the covered part when there is no start bound (owner-approved 2026-11-11) + +When the data starts inside an interval (no usable value before it) the baseline's interpolated start bound is `BadNoData`, so the region before the first value is Bad and left out — Part 13 §5.4.3.6/§5.4.3.8 "No Start Bound: Calculate Partial Interval", and what the HistoryRead evaluator (`TimeWeightedAggregator`, SqlServer + QuestDB) does. But when that first value is **Uncertain**, `AggregateCalculator.GetRegionsInValueSet` (ignoreBadData, sloped) overwrites the region's Bad status with `UncertainDataSubNormal` (the one branch without the not-Bad guard), so `AverageAggregateCalculator` integrated the uncovered span as a ramp from 0 and divided TimeAverage by the whole interval (QP: Total 188.96 for 163.96, TimeAverage 9.448 for 10.931). SDK 2.0.1's stock calculator does the same. ScadaBridge uses neither AggregateFilter nor these aggregates. + +- The copied upstream files above are **unchanged** (their SHA-256s still match `baseline-sources.json`). +- `CoveredTimeAverageAggregateCalculator.cs` (application code) subclasses the baseline `AverageAggregateCalculator` and recomputes only TimeAverage / Total: the baseline's interpolated-bounds values and regions, with a region that starts at a Bad point (only the start bound can) kept Bad. Values change only for such an interval; status (Uncertain either way, except `BadNoData` when nothing at all is covered), timestamps, value type and Calculated / Partial bits are the baseline's. Stepped variables and `TreatUncertainAsBad = true` never reached the defect and are unchanged. `CompatibilityAggregateFactory.Create` routes exactly those two IDs to it; TimeAverage2 / Total2 and the other 31 IDs are the baseline unchanged. +- Golden test: **no** fixture vector changes — every fixture data set has a usable value at −10 s, before every window, so no fixture interval lacks a start bound; all 384 TimeAverage / Total vectors stay held to their own output (`Issue821FixtureExemptions = 0`). The change is pinned over late-start variants of every data set (cut at +5 s and +25 s, first remaining non-Bad sample made Uncertain, every direction / stepping / configuration / incremental mode): 150 and 142 of the 384 change, each held to a baseline-derived oracle — the untouched baseline's value on the same variant with that sample Good, its status / timestamps / bits on the variant itself. `Sdk2AggregateCompatibilityTests.Issue821_*` also pins a hand-computed case. +- Live-vs-history parity: `SdkAggregateParityTests.Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start` compares the registered calculator with `TimeWeightedAggregator`, forward and reverse, with no masks, and checks that the data discriminates against the pre-#821 baseline; `QualityMixConformance` now includes TimeAverage / Total over QP on both SQL backends. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs index 5f38bb1ad..2e4649322 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs @@ -31,8 +31,12 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// (owner-approved 2026-11-11) overrides exactly those two (GoodOnlyStatusAggregateCalculator), so live and /// HistoryRead now agree on them — pinned by /// SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples. -/// Other known live-vs-HistoryRead differences (e.g. #821, an Uncertain first sample in a late-start interval) are -/// documented in docs/Historian.md. +/// Likewise the frozen AverageAggregateCalculator integrated the span before an Uncertain first value +/// as a ramp from 0 when the data starts inside an interval (no start bound) — live TimeAverage / Total disagreed +/// with HistoryRead and Part 13 ("No Start Bound: Calculate Partial Interval"). #821 (owner-approved 2026-11-11) +/// overrides exactly those two (CoveredTimeAverageAggregateCalculator) — pinned by +/// SdkAggregateParityTests.Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start. +/// The remaining documented live-vs-HistoryRead differences are listed in docs/Historian.md. /// /// /// A request asking for UseServerCapabilitiesDefaults, or spelling out exactly these values, is served; diff --git a/tests/Drivers/Historian.Conformance/QualityMixConformance.cs b/tests/Drivers/Historian.Conformance/QualityMixConformance.cs index 572a6b303..e920668f2 100644 --- a/tests/Drivers/Historian.Conformance/QualityMixConformance.cs +++ b/tests/Drivers/Historian.Conformance/QualityMixConformance.cs @@ -237,15 +237,16 @@ public abstract class QualityMixConformance : ConformanceBase public async Task Processed_reads_over_partial_data_match_the_Part_13_oracle(double from, double to, double intervalS) { // QP's start-of-data and end-of-data intervals against the oracle, forward and reverse (#821). Left out, each a documented or pinned divergence: - // TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region), the min/max family (the SDK sets Partial on a Bad - // result), and TimeAverage / Total (pinned by Time_weighted_values_over_partial_data_cover_only_the_data below). + // TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region) and the min/max family (the SDK sets Partial on a Bad + // result). TimeAverage / Total are in since #821: the live calculator no longer integrates the span before QP's + // Uncertain first sample (the hand-computed values are pinned below as well). await B.SeedAsync("QP", QP); await AssertMatchesOracle("QP", Rounded(QP), from, to, intervalS, [ HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.DurationGood, HistoryAggregateType.DurationBad, HistoryAggregateType.Count, HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta, - HistoryAggregateType.Interpolative, + HistoryAggregateType.Interpolative, HistoryAggregateType.TimeAverage, HistoryAggregateType.Total, ]); } @@ -255,9 +256,9 @@ public abstract class QualityMixConformance : ConformanceBase // Part 13 §5.4.3.3/§5.4.3.8 over QP's [0,20) (data from +5): the interpolated-bounding signal is defined from +5 // only — 10 → 11 over [5,12) (area 73.5) and 11 → 11 + 8/13 over [12,20) (area 4·(22 + 8/13)) — so // Total = 73.5 + 88 + 32/13 and TimeAverage = Total ÷ 15 s; Uncertain (an Uncertain value used, not fully covered), - // Partial (the data starts inside). Divergence found by #810: the 1.5.378.182 baseline, whose first sample is - // Uncertain here, integrates the uncovered [0,5) as a ramp from 0 (Total +25) — so these are pinned by hand, not - // against the oracle. + // Partial (the data starts inside). Found by #810: the untouched 1.5.378.182 baseline, whose first sample is + // Uncertain here, integrates the uncovered [0,5) as a ramp from 0 (Total +25); since #821 the live calculator (the + // oracle) does not. await B.SeedAsync("QP", QP); var src = B.CreateDataSource(); const double total = 73.5 + 88 + 32d / 13; @@ -268,8 +269,10 @@ public abstract class QualityMixConformance : ConformanceBase ((double)avg.Value!).ShouldBe(total / 15, 1e-6); avg.StatusCode.ShouldBe(UncertainCalc | Partial); - var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single(); - Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence — if this moves, compare QP's TimeAverage/Total to the oracle"); + var live = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single(); + Convert.ToDouble(live.Value).ShouldBe(total, 1e-6, "live AggregateFilter equals HistoryRead (#821)"); + var before = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single(); + Convert.ToDouble(before.Value).ShouldBe(total + 25, 1e-6, "the untouched baseline (live before #821)"); } [Fact] @@ -277,8 +280,8 @@ public abstract class QualityMixConformance : ConformanceBase { // #821: the reverse-order variant of the late-start case above — (0,20] read 20 → 0. The same Part 13 integral and // average over the covered [5,20); Uncertain; NOT Partial (a reverse read keys start-of-data on the latest sample, - // which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The - // baseline's reverse answer carries the same +25 ramp divergence. + // which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The live + // calculator agrees since #821; the untouched baseline's reverse answer carries the same +25 ramp. await B.SeedAsync("QP", QP); var src = B.CreateDataSource(); const double total = 73.5 + 88 + 32d / 13; @@ -290,8 +293,10 @@ public abstract class QualityMixConformance : ConformanceBase ((double)avg.Value!).ShouldBe(total / 15, 1e-6); avg.StatusCode.ShouldBe(UncertainCalc); - var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single(); - Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence (reverse) — #821"); + var live = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single(); + Convert.ToDouble(live.Value).ShouldBe(total, 1e-6, "live AggregateFilter equals HistoryRead (reverse) — #821"); + var before = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single(); + Convert.ToDouble(before.Value).ShouldBe(total + 25, 1e-6, "the untouched baseline (live before #821, reverse)"); } private async Task AssertMatchesOracle(string tag, (DateTime, HistorianValue, uint)[] rows, double from, double to, diff --git a/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs b/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs index e1274e597..3d37eace5 100644 --- a/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs +++ b/tests/Drivers/Historian.Conformance/SdkAggregateOracle.cs @@ -28,8 +28,15 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance; /// GoodOnlyStatusAggregateCalculator (the same baseline region machinery — simple bounds, end-of-data 1 ms /// region, status and Partial bits — counting only Good regions), so / are /// exactly what a live AggregateFilter computes, for every aggregate. PercentBad / DurationBad count Bad regions only -/// (unchanged by .182). runs the untouched baseline (what live served before #809), for the -/// tests that pin that divergence. +/// (unchanged by .182). runs the untouched baseline (what live served before #809 / #821), +/// for the tests that pin those divergences. +/// +/// +/// TimeAverage / Total with no start bound (#821). When the data starts inside an interval with an +/// Uncertain first value, the baseline integrates the uncovered span before it as a ramp from 0 (its region +/// machinery overwrites that region's Bad status with Uncertain). Part 13 ("No Start Bound: Calculate Partial +/// Interval") and ours integrate only the covered part, as the baseline itself does after a Good first value. Since +/// #821 the live calculators follow Part 13 here too (CoveredTimeAverageAggregateCalculator). /// /// /// Not 100 − PercentBad under TreatUncertainAsBad = true: that flag also changes the region machinery @@ -56,8 +63,8 @@ public static class SdkAggregateOracle /// /// Runs the Part 13 oracle for over : the live calculator - /// () — the baseline, except PercentGood / DurationGood, which count Good regions only (see the - /// remarks, #796/#820/#809). + /// () — the baseline, except PercentGood / DurationGood, which count Good regions only, and + /// TimeAverage / Total, which leave the span before an Uncertain first value out (see the remarks, #796/#820/#809/#821). /// /// The aggregate. /// Request start (later than for a reverse read). @@ -71,7 +78,8 @@ public static class SdkAggregateOracle /// /// Runs exactly the calculator the server registers for live AggregateFilter monitored items - /// (CompatibilityAggregateFactory.Create, the #809 Good-only PercentGood / DurationGood included). + /// (CompatibilityAggregateFactory.Create, the #809 Good-only PercentGood / DurationGood and the #821 + /// covered-only TimeAverage / Total included). /// /// The aggregate. /// Request start (later than for a reverse read). @@ -89,7 +97,8 @@ public static class SdkAggregateOracle /// /// Runs the untouched baseline (1.5.378.182) calculator for exactly as it computes it, /// under — including its PercentGood / DurationGood rule (Uncertain counted as - /// Good) that neither HistoryRead nor, since #809, the live calculators use. + /// Good) and its TimeAverage / Total ramp before an Uncertain first value, which neither HistoryRead nor, since + /// #809 / #821, the live calculators use. /// /// The aggregate. /// Request start (later than for a reverse read). diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs index eb25f5cd5..40aac4a41 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs @@ -525,17 +525,19 @@ public sealed class SdkAggregateParityTests public void An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data( double from, double to) { - // #821, pinned divergence (pending the #821 owner decision — this test records, it does not choose). - // Part 13 §5.4.3.3/§5.4.3.8: the interpolated-bounding signal is defined from the first sample only — 10 → 11 over - // [5,12) (73.5) and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s. + // #821, decided 2026-11-11: live follows Part 13. Part 13 §5.4.3.6/§5.4.3.8 ("No Start Bound: Calculate Partial + // Interval"): the interpolated-bounding signal is defined from the first sample only — 10 → 11 over [5,12) (73.5) + // and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s. const double part13Total = 73.5 + 88 + 32d / 13; Pct(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9); Pct(Ours(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9); + // The live AggregateFilter calculators (CompatibilityAggregateFactory — CoveredTimeAverageAggregateCalculator since + // #821; Sdk2AggregateCompatibilityTests proves the runtime manager serves the same code) answer the same. + Pct(Sdk(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9); + Pct(Sdk(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9); - // The frozen 1.5.378.182 calculators — for Total / TimeAverage exactly what the server registers for live - // AggregateFilter items (CompatibilityAggregateFactory; Sdk2AggregateCompatibilityTests proves the runtime manager - // serves the same code) - // — integrate the uncovered [0,5) as a ramp 0 → 10 (+25) and divide TimeAverage by the WHOLE 20 s interval. + // The untouched frozen 1.5.378.182 calculators (live before #821) integrate the uncovered [0,5) as a ramp 0 → 10 + // (+25) and divide TimeAverage by the WHOLE 20 s interval. Pct(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart)).ShouldBe(part13Total + 25, 1e-9); Pct(AsIs(HistoryAggregateType.TimeAverage, from, to, UncertainFirstLateStart)).ShouldBe((part13Total + 25) / 20, 1e-9); // SDK 2.0.1's own stock calculators answer the same: the upgrade did not change this. @@ -546,15 +548,18 @@ public sealed class SdkAggregateParityTests // Status agrees everywhere (Uncertain — an Uncertain value is used and the interval is not fully covered). (AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask) .ShouldBe(HistorianStatusCodes.UncertainDataSubNormal); + Sdk(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode + .ShouldBe(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode); (Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask) .ShouldBe(HistorianStatusCodes.UncertainDataSubNormal); - // Control: the SAME data with a Good first sample — all three agree on the Part 13 answer, so the trigger is the + // Control: the SAME data with a Good first sample — all agree on the Part 13 answer, so the trigger is the // Uncertain first sample, not the late start. foreach (var agg in new[] { HistoryAggregateType.Total, HistoryAggregateType.TimeAverage }) { var expected = agg == HistoryAggregateType.Total ? part13Total : part13Total / 15; Pct(Ours(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9); + Pct(Sdk(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9); Pct(AsIs(agg, from, to, GoodFirstLateStart)).ShouldBe(expected, 1e-9); Stock201(agg, from, to, GoodFirstLateStart).Single().Value!.Value.ShouldBe(expected, 1e-9); } @@ -661,6 +666,65 @@ public sealed class SdkAggregateParityTests .Select(r => Convert.ToDouble(r.Value)).ShouldBe([7000d, 10000, 10000, 10000, 5000, 10000, 10000, 7000], tolerance: 1e-9); } + /// + /// #821: late-start data whose first usable sample is Uncertain (+7, after every window's start), with Good / Uncertain / Bad samples after it + /// and a Good sample past every window's end (+90), so only the start of the data is in question. No sample sits on + /// an interval edge of (the reverse on-edge divergence, class remarks). + /// + private static readonly StoredSample[] UncertainFirstLate = + [ + S(7, HistorianValue.FromDouble(10), Uncertain), + S(12, HistorianValue.FromDouble(11), Good), + S(25, HistorianValue.FromDouble(12), Uncertain), + S(33, HistorianValue.FromDouble(14), Good), + S(47, HistorianValue.FromDouble(13), Bad), + S(52, HistorianValue.FromDouble(15), Good), + S(90, HistorianValue.FromDouble(16), Good), + ]; + + /// #821: the same, preceded by Bad samples only (an outage before the data — still no start bound). + private static readonly StoredSample[] BadThenUncertainFirstLate = + [ + S(-10, HistorianValue.Empty, BadNoCommunication), + S(2, HistorianValue.FromDouble(7), Bad), + .. UncertainFirstLate, + ]; + + [Theory] + [MemberData(nameof(WholeIntervalWindows))] + public void Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start( + double from, double to, double intervalS) + { + // #821: a live AggregateFilter monitored item and a HistoryRead-Processed (SqlServer / QuestDB share + // TimeWeightedAggregator) answer TimeAverage / Total identically when the data starts inside an interval with an + // Uncertain value — same values, timestamps, status and Partial / Calculated bits, NO documented-divergence mask. + // RunLive is exactly CompatibilityAggregateFactory.Create, the calculator the server registers. + var interval = intervalS > 0 ? TimeSpan.FromSeconds(intervalS) : (At(to) - At(from)).Duration(); + var failures = new List(); + var discriminated = 0; + foreach (var agg in new[] { HistoryAggregateType.TimeAverage, HistoryAggregateType.Total }) + foreach (var (name, data) in new[] { ("UncertainFirstLate", UncertainFirstLate), ("BadThenUncertainFirstLate", BadThenUncertainFirstLate) }) + { + var rows = data.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray(); + var live = SdkAggregateOracle.RunLive(agg, At(from), At(to), interval, rows); + try + { + SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), live, $"live {agg} {name} {from}→{to}/{intervalS}s"); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + // The data must discriminate: the pre-#821 live calculator (the untouched baseline) integrates the span before + // the data, so it disagrees on the late-start interval — or this parity would pass against the old behaviour too. + var before = SdkAggregateOracle.RunSdkAsIs(agg, At(from), At(to), interval, rows); + if (before.Zip(live).Any(p => Math.Abs(Convert.ToDouble(p.First.Value) - Convert.ToDouble(p.Second.Value)) > 1e-9)) + discriminated++; + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + discriminated.ShouldBe(4, "every aggregate × data set must differ from the pre-#821 live answer somewhere"); + } + /// #684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6). private static readonly StoredSample[] UncertainBeyond = [ diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs index 7132eaf93..5857950ab 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs @@ -6,10 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// #821 / #809 facts, through the BOOTED server's runtime AggregateManager — the calculators a live -/// AggregateFilter monitored item gets, under the server's own default configuration. The #821 Total / TimeAverage -/// facts pin current behaviour for the owner decision; they do not choose. #809 is decided (2026-11-11): live -/// PercentGood / DurationGood count Good time only, equal to HistoryRead. The HistoryRead (Part 13) answers and SDK -/// 2.0.1's stock answers for the same input are pinned beside it in SdkAggregateParityTests. +/// AggregateFilter monitored item gets, under the server's own default configuration. Both are decided (owner, +/// 2026-11-11): live PercentGood / DurationGood count Good time only (#809), and live Total / TimeAverage integrate only +/// the covered part of an interval the data starts inside, even with an Uncertain first sample (#821) — equal to +/// HistoryRead. The HistoryRead (Part 13) answers, the untouched baseline's and SDK 2.0.1's stock answers for the same +/// input are pinned beside it in SdkAggregateParityTests. /// public sealed class LiveAggregateLateStartFactsTests { @@ -41,11 +42,12 @@ public sealed class LiveAggregateLateStartFactsTests return d; } - // Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s. - // The live calculators integrate the uncovered [0,5) as a ramp from 0 (+25) and divide by the whole 20 s. + // #821: Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s + // — live as in HistoryRead (before #821 the live calculator integrated the uncovered [0,5) as a ramp from 0 (+25) + // and divided by the whole 20 s: 188.96… / 9.448…). const double part13Total = 73.5 + 88 + 32d / 13; - One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total + 25, 1e-9); // 188.96… - One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe((part13Total + 25) / 20, 1e-9); // 9.448… (Part 13: 10.93…) + One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total, 1e-9); // 163.96… + One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe(part13Total / 15, 1e-9); // 10.93… // #809: the Uncertain [5,12) is not Good — only [12,20) is, live as in HistoryRead (8 000 ms / 40 %; before #809 // the live calculator counted the Uncertain time too: 15 000 ms / 75 %). One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(8_000d, 1e-9); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs index e40367fae..f87e13b85 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs @@ -15,6 +15,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// case computed by the baseline SDK with TreatUncertainAsBad = true (its "twin": there 1.5.378.182 already /// counted Good regions only) — and each is pinned to that twin instead. The twin is baseline evidence, not output of /// the application's calculator, so the pin never regenerates expectations from the implementation. +/// +/// #821 (owner-approved 2026-11-11): TimeAverage / Total no longer integrate the span before an Uncertain first value +/// when there is no start bound. No fixture vector changes (every data set has a usable value at −10 s, before every +/// window, so no fixture interval lacks a start bound) — all 384 TimeAverage / Total vectors stay held to their own +/// output. The change is pinned instead over late-start variants of every data set against a baseline-derived oracle +/// () and by hand. +/// /// public sealed class Sdk2AggregateCompatibilityTests { @@ -22,6 +29,12 @@ public sealed class Sdk2AggregateCompatibilityTests private const string DurationGood = "AggregateFunction_DurationGood"; private const int Issue809PercentGoodExemptions = 38; private const int Issue809DurationGoodExemptions = 26; + private const string TimeAverage = "AggregateFunction_TimeAverage"; + private const string Total = "AggregateFunction_Total"; + /// #821: fixture vectors changed — none (every data set has a usable value before the window; class remarks). + private const int Issue821FixtureExemptions = 0; + private const int Issue821Cut5Changed = 150; + private const int Issue821Cut25Changed = 142; [Theory] [InlineData(false)] @@ -44,6 +57,8 @@ public sealed class Sdk2AggregateCompatibilityTests baseline.Vectors.Select(x => x.Aggregate).Distinct().Count().ShouldBe(37); exemptions.Keys.Count(x => x.Aggregate == PercentGood).ShouldBe(Issue809PercentGoodExemptions); exemptions.Keys.Count(x => x.Aggregate == DurationGood).ShouldBe(Issue809DurationGoodExemptions); + // #821: no TimeAverage / Total fixture vector is exempt — all 384 are held to their own baseline output above. + exemptions.Keys.Count(x => x.Aggregate is TimeAverage or Total).ShouldBe(Issue821FixtureExemptions); failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing (#809 exemptions: the uncertain-as-bad twin)."); } @@ -97,6 +112,104 @@ public sealed class Sdk2AggregateCompatibilityTests output.Select(x => double.Parse(x.Payload!, CultureInfo.InvariantCulture)).ShouldBe(expected); } + /// + /// #821 pin over late-start variants of every fixture data set (the fixture itself has none — see the class remarks): + /// each TimeAverage / Total vector's data set cut at (no usable value before the data + /// starts, so no start bound) with its first remaining non-Bad sample made Uncertain, run through the application's + /// calculator under the vector's own direction, stepping, configuration and incremental draining. + /// + /// The oracle is the untouched BASELINE, never the application's calculator: under TreatUncertainAsBad = false + /// the baseline treats an Uncertain value exactly like a Good one except for statuses, and with a Good first + /// value it already leaves the uncovered span out (the #821 control). So the expected value of each interval is + /// the baseline's on the same variant with that first sample Good, and the expected status, timestamps and bits + /// the baseline's on the variant itself (the region before the data is non-Good either way) — except an + /// interval the Good twin answers Bad (nothing covered: only the uncovered span ever fed it), which is the twin + /// whole. Stepped variables and TreatUncertainAsBad = true never reached the baseline's defect and must be the + /// baseline byte for byte. + /// + /// + [Theory] + [InlineData(5, Issue821Cut5Changed)] + [InlineData(25, Issue821Cut25Changed)] + public void Issue821_late_start_with_an_uncertain_first_sample_integrates_only_the_covered_part(double cutSeconds, int expectedChanged) + { + var baseline = LoadBaseline(); + var failures = new List(); + var changed = 0; + // Why the fixture needs no #821 exemption: every data set with a usable value has one before the window (−10 s), so + // no fixture interval lacks a start bound; the variants below remove it. + baseline.Data.Values.Where(d => d.Any(s => !StatusCode.IsBad(s.Status))) + .ShouldAllBe(d => d.Any(s => s.Seconds < 0 && !StatusCode.IsBad(s.Status))); + var vectors = baseline.Vectors.Where(v => v.Aggregate is TimeAverage or Total).ToList(); + foreach (var vector in vectors) + { + var (variant, goodTwin) = LateStartVariant(baseline.Data[vector.DataSet], cutSeconds); + var asIs = Run(baseline, vector, rig: null, out var asIsError, variant, baselineAsIs: true); + var twin = Run(baseline, vector, rig: null, out var twinError, goodTwin, baselineAsIs: true); + var actual = Run(baseline, vector, rig: null, out var error, variant); + asIsError.ShouldBeNull(Describe(vector)); + twinError.ShouldBeNull(Describe(vector)); + twin.Count.ShouldBe(asIs.Count, Describe(vector)); + + Value[] expected = EffectiveTreatUncertainAsBad(vector) || vector.Stepped + ? [.. asIs] + : [.. asIs.Zip(twin, (a, g) => + { + g.SourceTicks.ShouldBe(a.SourceTicks, Describe(vector)); + return StatusCode.IsBad(g.Status) ? g : a with { Type = g.Type, Payload = g.Payload }; + })]; + if (!expected.SequenceEqual(asIs)) changed++; + if (error is not null || !actual.SequenceEqual(expected)) + failures.Add($"{Describe(vector)} cut {cutSeconds}: error {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(actual.ToArray())}"); + } + vectors.Count.ShouldBe(384); + failures.ShouldBeEmpty("Live TimeAverage / Total must leave the span before an Uncertain first value out (Part 13 'No Start Bound: Calculate Partial Interval', #821)."); + changed.ShouldBe(expectedChanged, "the late-start variants the #821 override changes from the untouched baseline"); + } + + /// + /// #821 hand-computed pin, independent of every calculator: the "uncertain" data set from +5 (Uncertain 100 at 5, + /// Good 18 at 15, Bad at 25, Good 19 at 35, …), forward, sloped, [0,20). Covered [5,20): 100 → 18 over 10 s = 590, + /// then 18 → the bound at 20 (sloped 18 → 19 across the Bad sample: 18.25) over 5 s = 90.625 ⇒ Total 680.625, + /// TimeAverage 680.625 ÷ 15 = 45.375. The baseline added the ramp 0 → 100 over [0,5) (+250): 930.625 and 46.53125. + /// Uncertain either way (an Uncertain value is used, a Bad one skipped, the interval not covered). + /// + [Theory] + [InlineData(Total, 680.625, 930.625)] + [InlineData(TimeAverage, 45.375, 46.53125)] + public void Issue821_late_start_hand_computed(string aggregate, double part13, double baselineValue) + { + var baseline = LoadBaseline(); + var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped + && !v.UncertainAsBad && !v.Incremental && v.Configuration is null); + var (variant, _) = LateStartVariant(baseline.Data["uncertain"], 5); + variant[0].Status.ShouldBe(0x40A40000u, "the data set's own Uncertain 100 at +5"); + var output = Run(baseline, vector, rig: null, out var error, variant); + var asIs = Run(baseline, vector, rig: null, out _, variant, baselineAsIs: true); + error.ShouldBeNull(); + double.Parse(output[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(part13, 1e-9); + double.Parse(asIs[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(baselineValue, 1e-9); + (output[0].Status & 0xFFFF0000u).ShouldBe(StatusCodes.UncertainDataSubNormal.Code); + output[0].Status.ShouldBe(asIs[0].Status); + output.Skip(1).ShouldBe(asIs.Skip(1), "only the late-start interval changes"); + } + + /// + /// A data set cut at with its first non-Bad sample Uncertain (kept Uncertain if it already + /// is), and the same with that sample Good. + /// + private static (Sample[] Variant, Sample[] GoodTwin) LateStartVariant(Sample[] data, double cutSeconds) + { + var cut = data.Where(s => s.Seconds >= cutSeconds).OrderBy(s => s.Seconds).ToArray(); + var first = Array.FindIndex(cut, s => !StatusCode.IsBad(s.Status)); + if (first < 0) return (cut, cut); + var variant = cut.ToArray(); + var good = cut.ToArray(); + variant[first] = cut[first] with { Status = StatusCode.IsUncertain(cut[first].Status) ? cut[first].Status : 0x40A40000u }; + good[first] = cut[first] with { Status = 0u }; + return (variant, good); + } + private static Baseline LoadBaseline() => JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; @@ -126,7 +239,8 @@ public sealed class Sdk2AggregateCompatibilityTests : null; } - private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error) + private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error, + Sample[]? data = null, bool baselineAsIs = false) { var telemetry = DefaultTelemetry.Create(_ => { }); var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc); @@ -142,13 +256,15 @@ public sealed class Sdk2AggregateCompatibilityTests }; var calculator = rig is not null ? rig.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config) - : CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry); + : baselineAsIs + ? new BaselineAdapter(Aggregators.CreateStandardCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry)!) + : CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry); calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}."); var output = new RunOutput(); error = null; try { - var samples = baseline.Data[vector.DataSet].OrderBy(x => x.Seconds); + var samples = (data ?? baseline.Data[vector.DataSet]).OrderBy(x => x.Seconds); foreach (var sample in vector.Reverse ? samples.Reverse() : samples) { var time = epoch.AddSeconds(sample.Seconds); @@ -185,6 +301,20 @@ public sealed class Sdk2AggregateCompatibilityTests "Double" => double.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Int64" => long.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Boolean" => bool.Parse(sample.Payload!), _ => null }; static string? Format(object? value) => value switch { null => null, double number => number.ToString("R", CultureInfo.InvariantCulture), float number => number.ToString("R", CultureInfo.InvariantCulture), IFormattable f => f.ToString(null, CultureInfo.InvariantCulture), _ => value.ToString() }; private sealed class RunOutput : List { public List Accepted { get; } = []; } + + /// The untouched baseline calculator behind the SDK 2 protocol (what live served before #809 / #821). + private sealed class BaselineAdapter(AggregateCalculator calculator) : Opc.Ua.Server.IAggregateCalculator + { + public NodeId AggregateId => calculator.AggregateId; + public bool QueueRawValue(DataValue value) => calculator.QueueRawValue(LegacyDataValue.FromSdk(value)); + public bool HasEndTimePassed(DateTimeUtc currentTime) => calculator.HasEndTimePassed((DateTime)currentTime); + public bool TryGetProcessedValue(bool returnPartial, out DataValue value) + { + var processed = calculator.GetProcessedValue(returnPartial); + value = processed is null ? default : processed.ToSdk(); + return processed is not null; + } + } record Baseline(long EpochTicks, int IntervalMilliseconds, Dictionary Data, Vector[] Vectors); record Sample(double Seconds,string Type,string? Payload,uint Status,ushort SourcePicoseconds = 0,ushort ServerPicoseconds = 0,long ServerOffsetTicks = 0); record Value(bool PartialRequested,string? Type,string? Payload,uint Status,long SourceTicks,long ServerTicks,ushort SourcePicoseconds,ushort ServerPicoseconds); From d2b1545659bd1420764ce16d1fcee76eece0bd61 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:31:02 -0500 Subject: [PATCH 13/28] =?UTF-8?q?fix(opcua):=20#855=20review=201=20?= =?UTF-8?q?=E2=80=94=20re-decide=20frozen=20replay=20entries;=20a=20failed?= =?UTF-8?q?=20delivery=20is=20not=20a=20deferral?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - QueueFrozenReplay re-asks each item's verdict right before each entry's gate hold (Server-object and folder cut paths): a grant revoked between the pre-Lock verdicts and the replay no longer lets the replay through. A denied entry is also removed from the cut's re-sent set. - ReportEventWithoutBlocking logs and returns on a delivery that faulted inline instead of booking it as a deferral episode (which ran the drain and ClearStaleRefreshCuts under the event-report gate while Lock was held). - OnConditionReportEvent isolates failures per Server-object item. - Tests: revocation between cut and replay (folder + Server object), throwing hop not booked as a deferral; production-overload sampling inside Lock. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- docs/security.md | 6 +- .../OtOpcUaNodeManager.cs | 79 ++++++++++-- .../FolderConditionRefreshWireTests.cs | 112 +++++++++++++++++- 3 files changed, 182 insertions(+), 15 deletions(-) diff --git a/docs/security.md b/docs/security.md index 1e09644b1..1d02687b5 100644 --- a/docs/security.md +++ b/docs/security.md @@ -924,7 +924,11 @@ Behaviours worth knowing before enabling it: `QueueSize`) and the verdict runs on the reporter's thread, which already holds `Lock` — so a burst under one `Lock` hold completes with nothing deferred, and the #838 deadlock shape is gone for condition events. Folder items therefore see condition events in report order. The cost moves with it: the per-event verdict and filter for - folder items now run under `Lock` (as for Server-object items), where the SDK used to cache the verdict off it. + folder items now run under `Lock` (as for Server-object items), where the SDK used to cache the verdict off it — + including the SDK item queue's own per-event costs (its duplicate scan, and at a full queue a list shift), so many + folder items on one notifier whose clients stop publishing make each report proportionally slower under `Lock`. + With auditing on (off in this repo), a condition's SDK method **audit** event still takes the folder's channel, so + at a folder item it may now arrive after the condition event that method produced (they were one FIFO before). - **The deferral queue and the cut stay as the safety net**, for a hop that is not ours (none exists today: conditions are never event notifiers and variables cannot host event items) — and folder items share the cut: a `ConditionRefresh`/`ConditionRefresh2` of a folder item takes the same two paths as the Server object (plain walk diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index c3d96cc82..56e116e51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -1529,6 +1529,16 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 if (delivery.IsCompletedSuccessfully) return; + // #855 review: a delivery that FAILED inline is over, not suspended. Booking it as a deferral used to open and close + // a deferral episode synchronously (moving the episode a concurrent plain refresh checks, and clearing cuts under + // the event-report gate while Lock was held). Only an incomplete delivery is deferred. + if (delivery.IsCompleted) + { + _logger.LogError(delivery.AsTask().Exception?.GetBaseException(), + "OtOpcUaNodeManager: failed to report Part 9 condition event for {NodeId}", condition.NodeId); + return; + } + lock (_eventReportGate) { var queue = new DeferredConditionReports(); @@ -1901,6 +1911,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// private long _deferralEpisode; + /// Deferral episodes ended since start (test observation, #855 review). + internal long DeferralEpisodes => Interlocked.Read(ref _deferralEpisode); + /// What a Server-object item's latest ConditionRefresh conveyed (#854). /// Every condition event with a sequence at or below this was reported before the replay was read. /// The deferral episode the cut was taken in; once it ends the cut is stale and inert. @@ -5782,17 +5795,26 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 foreach (var item in _allEventsItems.Keys) { - if (e is AuditEventState && !IsAuditDeliverableTo(item)) continue; - if (scopes is not null && !IsConditionVisibleTo(item, scopes)) continue; - if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue; - if (reportingSession is { IsNull: false } reportingId - && item.Session?.Id is { } own - && !own.Equals(reportingId)) + try { - continue; - } + if (e is AuditEventState && !IsAuditDeliverableTo(item)) continue; + if (scopes is not null && !IsConditionVisibleTo(item, scopes)) continue; + if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue; + if (reportingSession is { IsNull: false } reportingId + && item.Session?.Id is { } own + && !own.Equals(reportingId)) + { + continue; + } - QueueConditionEventToItem(item, e, deferred, DeferredServerObjectDeliveryProbe); + QueueConditionEventToItem(item, e, deferred, DeferredServerObjectDeliveryProbe); + } + catch (Exception ex) + { + // #855 review: one item's failure (an item disposed under us, the SDK's operation guard at shutdown) must + // not cost the other Server-object items, nor abort the walk before the folder hops. + _logger.LogWarning(ex, "OtOpcUaNodeManager: failed to deliver a condition event to Server-object item {ItemId}", item.Id); + } } } @@ -6671,7 +6693,24 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } ServerObjectRefreshProbe?.Invoke(true, Monitor.IsEntered(Lock)); - QueueFrozenReplay(items, states, perItem.Select(entries => entries.Select(x => (x.Event, x.Condition.NodeId)).ToList()).ToArray(), generation); + // #855 review: the verdicts above were taken before Lock; a deploy may have revoked a grant since. Each replay entry + // is re-decided right before it is queued (scopes resolved once per condition). + var scopeCache = new Dictionary>(); + QueueFrozenReplay( + items, states, perItem.Select(entries => entries.Select(x => (x.Event, x.Condition.NodeId)).ToList()).ToArray(), + generation, + (item, e) => + { + if (enforced) + { + if (ConditionOf(e) is not { } condition) return false; + if (!scopeCache.TryGetValue(condition.NodeId, out var scopes)) + scopeCache[condition.NodeId] = scopes = ConditionVisibilityScopes(condition); + if (!IsConditionVisibleTo(item, scopes)) return false; + } + + return !ServiceResult.IsBad(ValidateEventRolePermissions(item, e)); + }); } finally { @@ -6685,8 +6724,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 /// replay of its condition, so that replay entry is dropped rather than queued after it (the newer event is already /// inside RefreshStart..RefreshEnd). A later refresh of the item supersedes this one. /// + /// + /// The item's verdict for one entry, asked right before the entry's gate hold (never under it: it takes Lock), + /// against the CURRENT generation — the verdicts the cut was built on were taken earlier (#855 review). An entry + /// denied here is dropped AND removed from the cut's re-sent set, so the condition counts as "omitted for the + /// grants" and its later events are left to the ACL rather than withheld as superseded. + /// private static void QueueFrozenReplay( - List items, List states, List<(IFilterTarget Event, NodeId ConditionId)>[] perItem, int generation) + List items, List states, List<(IFilterTarget Event, NodeId ConditionId)>[] perItem, + int generation, Func allowed) { for (var i = 0; i < items.Count; i++) { @@ -6694,9 +6740,17 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 var state = states[i]; foreach (var (e, conditionId) in perItem[i]) { + if (state.PendingReplay != generation) break; + var permitted = allowed(item, e); lock (state.Gate) { if (state.PendingReplay != generation) break; + if (!permitted) + { + state.Cut?.Refreshed.Remove(conditionId); + continue; + } + if (state.NewerSinceCut.Contains(conditionId)) continue; item.QueueEvent(e); } @@ -6926,7 +6980,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } DirectRefreshProbe?.Invoke(true, Monitor.IsEntered(Lock)); - QueueFrozenReplay(items, states, perItem, generation); + // #855 review: re-decided per entry right before queueing — a deploy may have revoked since the verdicts. + QueueFrozenReplay(items, states, perItem, generation, (item, e) => IsDirectRefreshEventAllowed(item, e, enforced)); } finally { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs index b01a93427..0a9bc98b9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs @@ -358,6 +358,110 @@ public sealed class FolderConditionRefreshWireTests keyEvents.Has(keyA, "after-revoke").ShouldBeFalse(); } + /// + /// #855 review: the cut path decides its replay before taking Lock; a grant revoked between that decision + /// and the queueing of the frozen replay must still withhold the replay (it is re-decided right before each entry). + /// The revocation lands from the refresh probe, which runs after the cut is installed and before the replay is queued. + /// + /// Refresh a Server-object item instead of a folder item. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_grant_revoked_between_the_cut_and_its_replay_withholds_the_replay(bool serverObject) + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855RevokeCut", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly); + var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant); + + // An anonymous subscriber keeps EqA a notifier fan-out the stall applies to (and stays granted throughout). + using var anonymous = await rig.ConnectAsync(anonymous: true, ct); + var anonEvents = new RecordedEvents(); + var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct); + RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA)); + await anonSub.ApplyChangesAsync(ct); + + using var keyed = await rig.ConnectAsync(anonymous: false, ct); + var keyEvents = new RecordedEvents(); + var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct); + var item = RecordedEvents.AddEventItem(keySub, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(keyed, EqA)); + await keySub.ApplyChangesAsync(ct); + item.Status.Error.ShouldBeNull(); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(item, "seed")); + + var revoked = 0; + void Revoke(bool cut, bool _) + { + if (!cut || Interlocked.Exchange(ref revoked, 1) != 0) return; + state.Apply( + ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(), + ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(), + ServerEventRig.TagReferences()); + } + + if (serverObject) rig.Nm.ServerObjectRefreshProbe = Revoke; + else rig.Nm.DirectRefreshProbe = Revoke; + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0); + await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id); + await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(item, ObjectTypeIds.RefreshEndEventType)); + stall.Release(); + } + + rig.Nm.ServerObjectRefreshProbe = null; + rig.Nm.DirectRefreshProbe = null; + Volatile.Read(ref revoked).ShouldBe(1, "the refresh took the cut path"); + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + + var (replayed, after) = Window(keyEvents.Of(item)); + replayed.ShouldBeEmpty("the grant was revoked before the replay was queued"); + after.ShouldBeEmpty("and nothing reaches the item after RefreshEnd either"); + } + + /// + /// #855 review: a delivery that fails inline (here: a hop that throws) is over, not suspended — it must not be booked + /// as a deferral episode (which moved the episode a concurrent plain refresh checks, and cleared cuts under the + /// event-report gate while Lock was held). The Server-object item, served before the failing hop, still gets + /// the event, and the next event is delivered normally. + /// + [Fact] + public async Task A_delivery_that_throws_is_not_booked_as_a_deferral() + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855Fault", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA)); + var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server); + await subscription.ApplyChangesAsync(ct); + + var episodes = rig.Nm.DeferralEpisodes; + rig.Nm.NotifierDeliveryStall = node => + node.NodeId.IdentifierAsString == EqA ? throw new InvalidOperationException("hop fault (test)") : null; + rig.Project(native: true, Active("faulted")); + rig.Nm.NotifierDeliveryStall = null; + + rig.Nm.DeferredEventReports.ShouldBe(0); + rig.Nm.DeferralEpisodes.ShouldBe(episodes, "a failed delivery is not a deferral episode"); + rig.Project(native: true, Active("next")); + await RecordedEvents.WaitUntilAsync(() => + events.Has(onServer, "faulted") && events.Has(onServer, "next") && events.Has(onA, "next")); + events.Has(onA, "faulted").ShouldBeFalse("the faulted hop delivered nothing"); + } + /// /// Production shape, no stall: bursts reported under one Lock hold (an apply pass) and single reports, against /// folder items that refresh repeatedly while their client is slow to publish. Folder items are served per item, so @@ -398,10 +502,14 @@ public sealed class FolderConditionRefreshWireTests { lock (rig.Nm.Lock) { - for (var i = 0; i < 1000; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); + for (var i = 0; i < 1000; i++) + { + rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); + // Sampled inside Lock, after each report: a deferral cannot open and drain unseen (#855 review). + peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports); + } } - peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports); rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); Thread.Sleep(2); } From cc1a7ec687f84e262d934d2fad756ac287c97359 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:31:51 -0500 Subject: [PATCH 14/28] fix(galaxy): #865 review round 1 - stale-session pump, data-path stream wait - EnsureEventPumpStarted replaces a pump created for an older registry session (the reopen -> replay window still held the dead session's pump, so an advise there lost its initial value and the replay skipped it); RestartEventPumpForReplay keeps a pump already on the current session. - SubscribeAsync, read-via-subscribe and ReplayAsync also await the bounded StreamRequested wait before advising, not just the probes. - TryEnsureEventPumpStarted tolerates a teardown flipping _runtimeStopping mid-check; the probe hook honours an already-cancelled seed token. - StreamRequested is documented as narrowing, not closing, the attach race. - Tests: reopen-window probe test, EventPump.StreamRequested tests, and the FixRound1 fresh-pump assertion now waits on a positive control. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 99 +++++++++++++++---- .../Runtime/EventPump.cs | 22 ++++- .../Runtime/EventPumpStreamRequestedTests.cs | 62 ++++++++++++ .../Runtime/GalaxyDriverFixRound1Tests.cs | 5 +- .../Runtime/ProbeInitialValueTests.cs | 45 +++++++++ 5 files changed, 209 insertions(+), 24 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpStreamRequestedTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 3102724f1..97a2d32cb 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -1053,6 +1053,10 @@ public sealed class GalaxyDriver // subscriptions to replay the dead pump used to stay in _eventPump, so every later // Subscribe/Read reused it and was silently deaf while health stayed Healthy. RestartEventPumpForReplay(); + // #865: every re-advise below pushes its initial value once, to whatever stream is attached. + EventPump? replayPump; + lock (_pumpLock) replayPump = _eventPump; + if (replayPump is not null) await AwaitStreamRequestedAsync(replayPump, cancellationToken).ConfigureAwait(false); // Only subscriptions still holding the previous session's handles: one registered after the // reopen is already live on the recreated session (re-subscribing it would double-advise). @@ -1308,6 +1312,10 @@ public sealed class GalaxyDriver { old = _eventPump; if (old is null) return; // pump never started — nothing to restart + // #865: a pump created after the reopen's fence (an advise in the reopen → replay window + // started it via EnsureEventPumpStarted) already streams the recreated session, and advises + // registered under it may be waiting for their initial values on it — keep it. + if (old.Session == _subscriptions.CurrentSession) return; _eventPump = null; } @@ -1681,7 +1689,10 @@ public sealed class GalaxyDriver var readSubscriber = _subscriber!; // A live pump must exist (values arrive through it); the read listens on ReadDataChange, not // on this instance, so it survives a reconnect's pump restart. - EnsureEventPumpStarted(); + var readPump = EnsureEventPumpStarted(); + // #865: a read waits ONLY for the initial value, which reaches only an already-requested stream. + // Not the caller's token: a cancel here must still yield per-ref Bad snapshots, and the wait is bounded. + await AwaitStreamRequestedAsync(readPump, CancellationToken.None).ConfigureAwait(false); var subscriptionId = _subscriptions.NextSubscriptionId(); // Pre-allocate one TaskCompletionSource per full-reference so the OnDataChange @@ -1927,6 +1938,8 @@ public sealed class GalaxyDriver var generation = Volatile.Read(ref _runtimeGeneration); var subscriber = _subscriber; var pump = EnsureEventPumpStarted(); + // #865: the advise's initial value reaches only an already-requested stream (no replay). + await AwaitStreamRequestedAsync(pump, cancellationToken).ConfigureAwait(false); var subscriptionId = _subscriptions.NextSubscriptionId(); if (fullReferences.Count == 0) @@ -2081,8 +2094,18 @@ public sealed class GalaxyDriver /// reverse map, not by spinning a pump per /// subscription. /// + /// + /// #865: a pump created for an OLDER registry session is replaced, not reused. Between + /// 's session swap and 's pump restart the + /// current pump still streams the dead gateway session; an advise made in that window goes to the + /// recreated session, so its initial value would reach no stream — and, registered under the + /// current session, the replay would not re-advise it either. A pump whose session is current + /// was created after the reopen's second fence, so its stream is on the recreated session. + /// private EventPump EnsureEventPumpStarted() { + EventPump? stale = null; + EventPump pump; lock (_pumpLock) { // Checked BEFORE the reuse: the flag is set before the teardown takes the pump under this @@ -2094,7 +2117,12 @@ public sealed class GalaxyDriver throw new InvalidOperationException( "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync."); } - if (_eventPump is not null) return _eventPump; + if (_eventPump is not null && _eventPump.Session == _subscriptions.CurrentSession) return _eventPump; + if (_eventPump is not null) + { + stale = _eventPump; + stale.OnDataChange -= OnPumpDataChange; + } _eventPump = new EventPump( _subscriber!, _subscriptions, _logger, channelCapacity: _options.MxAccess.EventPumpChannelCapacity, @@ -2103,13 +2131,26 @@ public sealed class GalaxyDriver onReplayGap: OnEventPumpReplayGap); _eventPump.OnDataChange += OnPumpDataChange; _eventPump.Start(); - return _eventPump; + pump = _eventPump; } + + if (stale is not null) DisposeStalePumpInBackground(stale); + return pump; } + /// Dispose a replaced pump off the caller's path (its stream is dead; the dispose only joins its loops). + private void DisposeStalePumpInBackground(EventPump stale) => _ = Task.Run(async () => + { + try { await stale.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) + { + _logger.LogWarning(ex, "GalaxyDriver {InstanceId} stale-session EventPump dispose failed.", _driverInstanceId); + } + }); + /// /// for infrastructure callers (the probe watcher) that - /// must not fail because the runtime is going away: false — and no pump — while a teardown is in + /// must not fail because the runtime is going away: null — and no pump — while a teardown is in /// progress or there is no subscriber. /// /// The running (or just started) pump; null when none may be started. @@ -2118,36 +2159,58 @@ public sealed class GalaxyDriver lock (_pumpLock) { if (_runtimeStopping || _disposed || _subscriber is null) return null; - return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant); cannot throw here + try + { + return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant) + } + catch (InvalidOperationException) + { + // _runtimeStopping is written without _pumpLock; a teardown that flipped it between the + // check above and EnsureEventPumpStarted's own check is the same "no pump" answer. + return null; + } } } - /// Upper bound on how long a probe subscribe waits for the pump's StreamEvents call to be issued. - internal TimeSpan ProbeStreamRequestWait { get; set; } = TimeSpan.FromSeconds(5); + /// Upper bound on how long an advise waits for the pump's StreamEvents call to be started. + internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(5); /// - /// #865 — the probe watcher's pre-subscribe hook: start the pump (if this runtime may) and wait, - /// bounded, until its StreamEvents call has been issued, so the gateway has a subscriber to push - /// the probe's initial ScanState value to. A fresh stream (after_worker_sequence = 0) is never - /// replayed, so an initial value pushed before it attaches is lost — and ScanState rarely changes - /// again. On timeout the subscribe proceeds anyway (logged): a late stream is no worse than before. + /// #865 — wait, bounded, until has started its StreamEvents call, so an + /// advise made next has a stream for its initial value. A fresh stream (after_worker_sequence = 0) + /// is never replayed, so an initial value pushed before it attaches is lost — and a slow-changing + /// attribute (ScanState, a setpoint) may not publish again for a long time. On timeout the caller + /// proceeds (logged): a late stream is no worse than the pre-#865 behaviour. /// - private async Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken) + private async Task AwaitStreamRequestedAsync(EventPump pump, CancellationToken cancellationToken) { - var pump = TryEnsureEventPumpStarted(); - if (pump is null) return; try { - await pump.StreamRequested.WaitAsync(ProbeStreamRequestWait, cancellationToken).ConfigureAwait(false); + await pump.StreamRequested.WaitAsync(StreamRequestWait, cancellationToken).ConfigureAwait(false); } catch (TimeoutException) { _logger.LogWarning( - "GalaxyDriver {InstanceId} event stream was not requested within {Wait} — subscribing the platform probes anyway; their initial ScanState may be missed.", - _driverInstanceId, ProbeStreamRequestWait); + "GalaxyDriver {InstanceId} event stream was not started within {Wait} — advising anyway; initial values may be missed.", + _driverInstanceId, StreamRequestWait); } } + /// + /// #865 — the probe watcher's pre-subscribe hook: start the pump (if this runtime may) and wait for + /// its stream (). A probe's initial ScanState value is usually + /// its only event, and a driver with no authored tags never Reads/Subscribes, so without this its + /// pump — and every host status — never existed. + /// + private async Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken) + { + // A seed pass a teardown already cancelled must not start a pump after the teardown finished. + cancellationToken.ThrowIfCancellationRequested(); + var pump = TryEnsureEventPumpStarted(); + if (pump is null) return; + await AwaitStreamRequestedAsync(pump, cancellationToken).ConfigureAwait(false); + } + /// /// Replay-gap callback for the . The gateway told us it evicted /// events we can never receive, so every cached value may be stale by an unknown amount. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index c3640ab12..e56d8d57a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -148,15 +148,27 @@ internal sealed class EventPump : IAsyncDisposable private readonly int _session; + /// + /// The registry session this pump was created for (#865). A pump of an older session was opened + /// against a gateway session a reconnect replaced; its stream will never carry the new session's events. + /// + internal int Session => _session; + private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously); /// - /// Completes once the stream loop has issued its StreamEvents call (the first - /// MoveNextAsync runs the subscriber's iterator synchronously up to the RPC) — or once the - /// loop has ended without getting that far, so a waiter never hangs on a dead pump. #865: the - /// gateway does not replay a fresh stream, so a caller whose only expected event is the initial - /// value of an advise it is about to make waits on this first. + /// Completes once the stream loop has STARTED its StreamEvents call (the first + /// MoveNextAsync runs the subscriber's iterator synchronously up to the gRPC call creation) — + /// or once the loop has ended without getting that far, so a waiter never hangs on a dead pump. + /// #865: the gateway does not replay a fresh stream, so a caller whose only expected event is the + /// initial value of an advise it is about to make waits on this first. /// + /// + /// ⚠️ This NARROWS the stream-attach race, it does not close it: the HTTP/2 request is sent + /// asynchronously and the gateway attaches the subscriber after auth, concurrently with any other + /// RPC. In practice the attach wins (an advise's initial value needs a worker IPC round trip plus an + /// MXAccess advise), but the gateway offers no attach acknowledgement to await. + /// public Task StreamRequested => _streamRequested.Task; /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpStreamRequestedTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpStreamRequestedTests.cs new file mode 100644 index 000000000..d1e28e1d6 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpStreamRequestedTests.cs @@ -0,0 +1,62 @@ +using System.Runtime.CompilerServices; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// #865: — what advise paths wait on so an initial value is +/// never pushed before the stream is requested. It must complete once the RPC is started (before any +/// event), and must never strand a waiter on a pump whose loop died before getting that far. +/// +public sealed class EventPumpStreamRequestedTests +{ + private sealed class ScriptedSubscriber(Func> stream) : IGalaxySubscriber + { + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + => Task.FromResult>([]); + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; + + public IAsyncEnumerable StreamEventsAsync(CancellationToken cancellationToken) => stream(cancellationToken); + } + + /// A stream that is requested and then never yields: StreamRequested completes; the RPC started first. + [Fact] + public async Task Completes_once_the_stream_is_requested_before_any_event() + { + var rpcStarted = 0; + async IAsyncEnumerable Never([EnumeratorCancellation] CancellationToken ct) + { + Interlocked.Exchange(ref rpcStarted, 1); // the synchronous first step — where a real call is created + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + yield break; + } + + await using var pump = new EventPump(new ScriptedSubscriber(Never), new SubscriptionRegistry()); + pump.StreamRequested.IsCompleted.ShouldBeFalse("not before Start"); + pump.Start(); + + await TestWait.ForAsync(pump.StreamRequested, "StreamRequested must complete once the stream call is started"); + Volatile.Read(ref rpcStarted).ShouldBe(1, "StreamRequested must not precede the stream call"); + } + + /// A subscriber that throws before reaching the RPC still completes StreamRequested (no stranded waiter). + [Fact] + public async Task Completes_when_the_stream_fails_before_the_rpc() + { + var faulted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var pump = new EventPump( + new ScriptedSubscriber(_ => throw new InvalidOperationException("not connected")), + new SubscriptionRegistry(), + onStreamFault: ex => faulted.TrySetResult(ex)); + pump.Start(); + + await TestWait.ForAsync(pump.StreamRequested, "a dead pump must not strand a StreamRequested waiter"); + (await TestWait.ForAsync(faulted.Task, "positive control: the failure is still reported as a stream fault")) + .ShouldBeOfType(); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs index 2ca4698a7..e2339a751 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs @@ -333,7 +333,10 @@ public sealed class GalaxyDriverFixRound1Tests await driver.InitializeAsync("{}", CancellationToken.None); // #865: the new runtime's background probe seed starts a pump of its own (its advises need a live // stream) — but it must be a FRESH one, never the dead subscriber's. - driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "positive control: the new runtime's background seed must advise its platform"); + driver.EventPumpInstanceForTests.ShouldNotBeNull("the seed's probe advise must have started a pump") + .ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump"); } // ───────────────────────────── IMPORTANT 3: bounded gateway close ───────────────────────────── diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs index 2144c8f7f..81c986ab2 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs @@ -193,6 +193,51 @@ public sealed class ProbeInitialValueTests sub.DroppedInitialValues.ShouldBe(0); } + /// + /// Review finding: a probe advised in the reopen → replay window (the session was swapped, the old + /// pump still streams the dead session) must still get its initial value — the stale-session pump + /// is replaced before the advise, and the replay keeps the replacement (it already streams the + /// recreated session). Reusing the old pump bound the value to the old session, so it never routed, + /// and the replay skipped the platform (its handle is current-session). + /// + [Fact] + public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + platformSource: new FixedPlatformSource(Platforms)) + { + RecreateOverrideForTests = _ => Task.CompletedTask, + }; + await driver.InitializeAsync("{}", CancellationToken.None); + await TestWait.UntilAsync( + () => Platforms.All(p => StateOf(driver, p) == HostState.Running), + "positive control: the initial seed reports Running", + () => Describe(driver, sub)); + var oldPump = driver.EventPumpInstanceForTests; + + await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet + sub.ScanStateRunning = false; + await driver.ProbeWatcherForTests.ShouldNotBeNull() + .SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None); + + await TestWait.UntilAsync( + () => StateOf(driver, "LatePlatform") == HostState.Stopped, + "a probe advised in the reopen window must get its initial value", + () => Describe(driver, sub)); + var windowPump = driver.EventPumpInstanceForTests; + windowPump.ShouldNotBeSameAs(oldPump, "the stale-session pump must have been replaced before the advise"); + + await driver.InvokeReplayForTestAsync(CancellationToken.None); + driver.EventPumpInstanceForTests.ShouldBeSameAs(windowPump, "the replay keeps a pump already on the current session"); + await TestWait.UntilAsync( + () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped), + "the replay re-advises the stale probes and their initial values arrive on the kept pump", + () => Describe(driver, sub)); + StateOf(driver, "LatePlatform").ShouldBe(HostState.Stopped); + sub.DroppedInitialValues.ShouldBe(0); + } + /// /// A probe sync on a driver whose pump was never started (no platform source, e.g. the /// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just From 8f30c1e5212c1a52bc08b69c4dd914b502119b11 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:34:07 -0500 Subject: [PATCH 15/28] docs(script-runner): windev re-measure (#789) and Windows soft-clock test run (#842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §12.1 gains the 2026-11-11 windev (VMware VM) perf table: idle p50 581 us (MISS), kill latency after T+G 8-16 ms (PASS), kill both shards 2.12-2.14 s (MISS by ~6 %), worker RSS 160 MB compiled / 36 MB warm-cache (PASS under the revised 200 MB budget), host CPU +131 % vs InProcess (MISS, untuned). It also records the first Windows run of Category=ScriptWorker: 107/0/2 from C:\build, all 13 WorkerSoftClockTests cases (5 freezing a real worker with NtSuspendProcess) green with no code change. From a user-profile directory the restricted-identity isolation test fails because the restricted token cannot read the worker dll there (profile ACL), which is environment, not a defect. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- ...-09-out-of-process-script-runner-design.md | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/plans/2026-11-09-out-of-process-script-runner-design.md b/docs/plans/2026-11-09-out-of-process-script-runner-design.md index b48d3db87..f9bd8e713 100644 --- a/docs/plans/2026-11-09-out-of-process-script-runner-design.md +++ b/docs/plans/2026-11-09-out-of-process-script-runner-design.md @@ -884,6 +884,58 @@ Both misses → lmxopcua#789 (the §4.5 batching and per-eval allocation work). The host load timeout still counts the time a load waits behind other compiles → lmxopcua#790. +**windev re-measure, 2026-11-11 (lmxopcua#789, after the §12.2–§12.3 changes; budget revision applied).** +- **windev is a VM:** VMware20,1 on an Intel Xeon E5-2697 v4 @ 2.30 GHz, 36 logical CPUs, 32 GB RAM, Windows, + .NET 10.0.8. The box was otherwise idle (about 4 % total CPU before the run). The issue's "re-measure on a non-VM + box" request is therefore still open; these are VM figures. +- Run from `C:\build\otopcua-842` (a `git archive` of the branch head) with `-c Release`, `OTOPCUA_SCRIPT_PERF=1`, + 10 s phases, filtered to `Throughput_and_latency_against_the_accepted_gate` and `Restart_rss_and_kill_latency`. + The restart test ran twice; both figures are given. +- Framework-dependent worker built beside the tests (`dotnet exec`), as in the earlier windev rows. + +| Measure (K = 2 unless stated) | windev 2026-11-11 | §8.2 budget | Verdict | +|---|---:|---|---| +| Added latency p50, idle sequential | 581 µs (K=1 490, K=4 537; InProcess 134) | ≤ 150 µs | MISS | +| p99 at paced 10k/s | 3,220 µs (K=1 5,380, K=4 1,358) | ≤ 2 ms | MISS at K=2 (was 1,676 µs in §12.2) | +| Host CPU at 10k/s, vs InProcess | +131 % (raw 141 %, InProcess 10 %) | ≤ 25 % | MISS (not tuned in this task) | +| Closed loop ×64 | 54,152 evals/s, 0 not ok | gate ≥ 5,000 | PASS | +| Kill latency after T+G (5 runs × 2) | 16 / 11 / 10 / 11 / 12 and 16 / 12 / 8 / 11 / 12 ms | ≤ 100 ms | PASS | +| Kill both shards → every script's next correct result (1,000 cached) | 2,135 ms and 2,117 ms | ≤ 2,000 ms | MISS by about 6 % | +| Warm restart, 1,000 cached, fresh runner | 1,847 ms and 1,829 ms | ≤ 2,000 ms | PASS | +| Worker RSS after first eval, compiled (max of 2 shards) | 160 MB (both runs) | ≤ 200 MB + scripts | PASS | +| Worker RSS after first eval, warm PE cache, no compile | 36 MB (both runs) | ≤ 200 MB + scripts | PASS | +| Worker RSS, 500 scripts compiled per shard | 236 / 262 MB | — (200 MB + scripts) | — | +| Worker RSS, 500 scripts loaded from the cache per shard | 93 / 96 MB | — | — | +| 1,000 scripts, empty cache, all first results | 12,380 / 12,800 ms | — | — | + +Notes: +- Under the revised budget, RSS passes. Kill latency stays fixed since §12.2: the earlier 210 ms was compile time. +- Kill-both-shards is about 120 ms over the line. It includes the 250 ms first restart backoff and the warm-cache + load of 1,000 PEs. The warm-cache restart alone takes about 1.83 s on this VM, so the kill path adds only about + 300 ms on top. +- Idle p50 and host CPU are the overlapped-I/O path that §12.2 left open; CPU tuning is deliberately not part of + this re-measure. The K=2 p99 at 10k/s reads worse than §12.2's 1,676 µs. K=4 is 1,358 µs in the same run, so this + is likely VM scheduling noise on a single run, but it was not re-run to confirm. + +**Windows `ScriptWorker.Tests` run, 2026-11-11 (lmxopcua#842).** `--filter Category=ScriptWorker`, Debug, on the +same VM. This is the first Windows run of the #827 soft-clock tests, which freeze the worker with `NtSuspendProcess` +/ `NtResumeProcess`. +- From `C:\build\otopcua-842`: **107 passed, 0 failed, 2 skipped (109)**, 13 m 37 s, 0 build warnings. + - All 13 `WorkerSoftClockTests` cases passed, including the 5 that suspend a real worker process: stalled > T, + stalled past T+G, late-starting runaway, Shadow under a stall (two freezes), and a notice that arrives after the + transport deadline. No code change was needed. + - Skipped by design: `A_published_self_contained_worker_runs_as_its_own_process` (needs + `OTOPCUA_SCRIPTWORKER_TEST_PATH`, §12.1) and the off-Windows identity refusal. +- From `C:\Users\dohertj2\otopcua-842`: 106 / **1 failed** / 2. The failure is + `WorkerIsolationTests.A_worker_under_the_identity_runs_scripts_with_a_restricted_token_and_a_plain_one_does_not`: + "The application to execute does not exist: …\script-worker\ZB.MOM.WW.OtOpcUa.ScriptWorker.dll", exit + `0x80008081`. + - **Cause: the test environment, not the code.** The restricted worker token cannot read the worker dll inside a + user profile, whose ACL is SYSTEM, Administrators and the owner only. That is the access §6 intends. + - In production the installer grants the identity read+execute on `script-worker\` (§12.5). + - Run the Windows suite from a directory the `BUILTIN\Users` group can read, such as `C:\build`. The §12.5 green + run used that. + ### 12.2 lmxopcua#789 / #790 / #788 record (2026-11-10) **What cost the host CPU (#789).** With `DOTNET_ThreadPool_UnfairSemaphoreSpinLimit=0` a scratch probe showed the From 19c7b5d7e4c1468a2362e44d4308871793968f8c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:38:45 -0500 Subject: [PATCH 16/28] =?UTF-8?q?fix(controlplane):=20#868=20review=20?= =?UTF-8?q?=E2=80=94=20complete=20fully-acked=20recoveries,=20replay=20que?= =?UTF-8?q?ued=20acks=20before=20a=20deadline,=20retry=20failed=20ack=20pe?= =?UTF-8?q?rsists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on 02503a4a (ConfigPublishCoordinator): 1 (HIGH) A recovered deployment whose every expected ack was already in NodeDeploymentState (a seal whose SaveChanges failed, a fused node's own row) was never completed: inside the deadline it waited for acks that had landed, past it HandleDeadline's "race" guard returned without sealing — stuck in AwaitingApplyAcks, refusing every later deploy. ApplyRecovery now completes it (Seal / PartiallyFailed), and HandleDeadline completes a full ack set instead of returning. 2 (MED-HIGH) The past-deadline recovery of a populated expected set ran HandleDeadline inline before the stash replay, discarding on-time acks queued during the outage. It is now self-sent, behind the replay. The zero-rows past-deadline branch stays inline on purpose: no ack can count there, and a self-sent deadline would time out a queued dispatch of that same row right after publishing it. 3 (MED) A failed PersistNodeAck threw, restarted the actor and dropped the ack (driver-only nodes never write their own row and do not resend). Acks are now persisted before being counted, and a failed persist is re-queued on a timer. The class remarks document that the Loading stash is node-local (lost on a hand-over; queued dispatches are recovered from their rows). Minor: MarkReady/Become(Ready) only after the recovery applied; a throw while applying is a failed load (backoff) rather than a tight restart loop; a queued dispatch matching the recovered Dispatching row logs Info, not the "lost dispatch" Warning. Tests (ConfigPublishCoordinatorDbOutageTests): fully-acked recovery seals / partially fails inside and past the deadline (4 cases); queued ack applied before a past-deadline recovery; failed ack persist retried. All 6 fail on 02503a4a. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../Coordinators/ConfigPublishCoordinator.cs | 111 +++++++++++++++--- .../ConfigPublishCoordinatorDbOutageTests.cs | 111 ++++++++++++++++++ 2 files changed, 204 insertions(+), 18 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index c96b682a9..c2b6770e1 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -64,6 +64,18 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; /// during the outage is queued, never dropped. reports the /// loading state (and the last error) to the node's health endpoint as Degraded. /// +/// +/// The stash is node-local. A singleton hand-over while the coordinator is still Loading discards what it +/// queued (the old incarnation stops with its mailbox). That loses nothing durable: a queued dispatch's row is +/// Dispatching with no expected-ack rows, which the new incarnation's recovery (or AdminOps' lost-dispatch +/// closure) re-drives or closes; a queued ack is the one exception — its node does not resend, so that node is +/// named in the deadline's timeout reason. +/// +/// +/// An ack is held in memory only once it is persisted. NodeDeploymentState is the system of record +/// (driver-only nodes never write their own row), so a failed PersistNodeAck re-queues the ack on a timer +/// instead of throwing — a throw would restart the actor and drop it. +/// /// public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithUnboundedStash { @@ -86,6 +98,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private readonly ConfigPublishCoordinatorReadiness? _readiness; private int _failedLoads; + /// Deployments whose dispatch was queued while Loading (only to word the recovery log correctly). + private readonly HashSet _queuedDispatches = new(); + /// /// Where central→node commands are handed off (per-cluster mesh Phase 2), or /// to publish on the mediator directly. @@ -204,6 +219,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU "DispatchDeployment {Id} queued: the coordinator has not yet loaded its recovery state from the " + "ConfigDb ({Failed} failed attempt(s) so far); it is dispatched as soon as the load succeeds", msg.DeploymentId, _failedLoads); + _queuedDispatches.Add(msg.DeploymentId); Stash.Stash(); }); ReceiveAny(_ => Stash.Stash()); @@ -214,6 +230,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU { Receive(HandleDispatch); Receive(HandleAck); + Receive(m => HandleAck(m.Ack)); Receive(HandleDeadline); Receive(_ => { /* DPS subscribe confirmation */ }); // A late load outcome / retry tick cannot normally arrive once Ready; ignored, never stashed. @@ -296,26 +313,38 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU } /// - /// Applies the recovered state, switches to and replays every stashed message. The - /// recovery DECISIONS (re-dispatch, time out) run inline, before the replay, so a queued message for the - /// same deployment is judged against the recovered state rather than racing a self-sent one. + /// Applies the recovered state, then switches to and replays every stashed message. A + /// decision that must see the queued messages first (the deadline of a recovered deployment that is already + /// past it) is SELF-SENT, so it lands behind the replay; one that must precede them (re-driving a lost + /// dispatch, so a queued duplicate of it is recognised) runs inline. A throw while applying is treated as a + /// failed load — the actor stays Loading and retries on the backoff, never a tight restart loop. /// private void HandleRecoveryLoaded(RecoveryLoaded msg) { Timers.Cancel(LoadRetryTimerKey); - if (_failedLoads > 0) - _log.Info("ConfigPublishCoordinator loaded its recovery state after {Failed} failed attempt(s)", _failedLoads); - _failedLoads = 0; - Become(Ready); - _readiness?.MarkReady(); try { ApplyRecovery(msg.Inflight); } - finally + catch (Exception ex) { - Stash.UnstashAll(); + _log.Error(ex, "ConfigPublishCoordinator failed to apply its recovered state; retrying the load"); + Timers.Cancel(DeadlineTimerKey); + _current = null; + _expectedAcks.Clear(); + _acks.Clear(); + _degraded.Clear(); + HandleRecoveryLoadFailed(new RecoveryLoadFailed(ex)); + return; } + + if (_failedLoads > 0) + _log.Info("ConfigPublishCoordinator loaded its recovery state after {Failed} failed attempt(s)", _failedLoads); + _failedLoads = 0; + _queuedDispatches.Clear(); + Become(Ready); + _readiness?.MarkReady(); + Stash.UnstashAll(); } /// @@ -350,16 +379,25 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU // closed as TimedOut with a reason that names the condition. if (remaining > TimeSpan.Zero) { - _log.Warning( - "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + - "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + - "re-dispatching with {Remaining} of the apply deadline left", - _current, remaining); + if (_queuedDispatches.Contains(_current.Value)) + _log.Info( + "Coordinator dispatching deployment {Id}, queued while its recovery state was loading", + _current); + else + _log.Warning( + "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + + "re-dispatching with {Remaining} of the apply deadline left", + _current, remaining); HandleDispatch(new DispatchDeployment( _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); } else { + // Deliberately INLINE, unlike the populated-set deadline below: with no expected set no queued ack + // can count, so there is nothing to let the replay apply first — and a queued dispatch of this very + // row must meet the TimedOut row (terminal-state guard ignores it), not publish to every node and + // be timed out the moment a self-sent deadline caught up with it. HandleDeadline(new DeadlineElapsed(_current.Value)); } return; @@ -374,9 +412,20 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId)); } + // Every expected ack is already in the DB (a seal whose SaveChanges failed, or a fused node's own + // DriverHostActor row): finish it now. Waiting for the deadline would leave it in flight — the deadline + // can only time out an INCOMPLETE set. + if (_acks.Count == _expectedAcks.Count) + { + _log.Info("Coordinator recovered deployment {Id} with every ack already landed; completing it", _current); + CompleteFromAcks(); + return; + } + if (remaining <= TimeSpan.Zero) { - HandleDeadline(new DeadlineElapsed(_current.Value)); + // Self-sent, so acks queued during the load (which arrived in time) are applied first. + Self.Tell(new DeadlineElapsed(_current.Value)); return; } @@ -516,20 +565,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU return; } + // Persist FIRST: the row is the system of record (driver-only nodes never write their own, and nodes do not + // resend). A throw here used to restart the actor and drop the ack; it is re-queued instead. + try + { + PersistNodeAck(msg); + } + catch (Exception ex) + { + _log.Warning(ex, + "Could not persist the ApplyAck from {Node} for {Id}; retrying in {Delay}", + msg.NodeId, msg.DeploymentId, _loadRetryMin); + Timers.StartSingleTimer(AckRetryTimerKey(msg), new RetryAckPersist(msg), _loadRetryMin); + return; + } + _acks[msg.NodeId] = msg.Outcome; // An old node never sets Degraded, so its ack takes the clean branch exactly as before. if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId); else _degraded.Remove(msg.NodeId); - PersistNodeAck(msg); if (_acks.Count < _expectedAcks.Count) return; + CompleteFromAcks(); + } + /// Seals (all Applied) or marks partially failed — called once every expected ack is in. + private void CompleteFromAcks() + { if (_acks.Values.All(o => o == ApplyAckOutcome.Applied)) SealDeployment(); else MarkPartiallyFailed(); } + private static string AckRetryTimerKey(ApplyAck msg) => $"ack-retry:{msg.DeploymentId.Value:N}:{msg.NodeId.Value}"; + private void PersistNodeAck(ApplyAck msg) { using var db = _dbFactory.CreateDbContext(); @@ -612,7 +682,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped. if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count) { - // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere. + // Every ack is in but nothing completed it (a completion normally resets _current, so the stale-id + // check above catches the ack-vs-deadline race). Complete it rather than leave it in flight forever. + CompleteFromAcks(); return; } @@ -682,6 +754,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private const string DeadlineTimerKey = "apply-deadline"; private const string LoadRetryTimerKey = "recovery-load-retry"; + /// An ack whose persist failed, re-delivered by its retry timer. + private sealed record RetryAckPersist(ApplyAck Ack); + public sealed record DeadlineElapsed(DeploymentId DeploymentId); /// The recovery read's result: the newest in-flight deployment, or null when none. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs index 0e702382a..ae10d1966 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs @@ -115,6 +115,105 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); } + /// + /// Review finding 1: a recovered deployment whose every expected ack is already in the DB (a seal whose + /// SaveChanges failed, or a fused node's own row) must be completed — inside the deadline it used to wait for + /// acks that had already landed, past it the deadline's "race" guard returned without sealing; either way it + /// sat in AwaitingApplyAcks forever and AdminOps refused every later deploy. + /// + [Theory] + [InlineData(false, false, DeploymentStatus.Sealed)] + [InlineData(true, false, DeploymentStatus.Sealed)] + [InlineData(false, true, DeploymentStatus.PartiallyFailed)] + [InlineData(true, true, DeploymentStatus.PartiallyFailed)] + public void Recovered_deployment_with_every_ack_already_persisted_is_completed(bool pastDeadline, bool oneFailed, DeploymentStatus expected) + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, Node, "site-a-2:4053"); + var deadline = TimeSpan.FromMinutes(1); + var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, + pastDeadline ? DateTime.UtcNow.AddMinutes(-10) : DateTime.UtcNow); + SeedNodeState(dbFactory, deploymentId, Node, NodeDeploymentStatus.Applied); + SeedNodeState(dbFactory, deploymentId, "site-a-2:4053", oneFailed ? NodeDeploymentStatus.Failed : NodeDeploymentStatus.Applied); + + Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(expected); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Review finding 2: a recovered deployment that is already past its deadline must first apply the acks that + /// were queued while the load was failing (they arrived in time); the deadline used to run inline before the + /// replay, so they were discarded as stale and the deployment was timed out. + /// + [Fact] + public void Ack_queued_during_the_outage_is_applied_before_a_past_deadline_recovery_times_out() + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true }; + SeedNodes(direct, Node); + var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow.AddMinutes(-10)); + SeedNodeState(direct, deploymentId, Node, NodeDeploymentStatus.Applying); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, TimeSpan.FromMinutes(1), loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2), + duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + outage.Down = false; + + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Review finding 3: an ack whose persist fails (ConfigDb blip while Ready) is retried, not dropped. It used to + /// throw, restart the actor and lose the ack — the node never resends and never writes its own row. + /// + [Fact] + public void Ack_whose_persist_fails_is_retried_not_dropped() + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(direct, Node); + var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow); + var readiness = new ConfigPublishCoordinatorReadiness(); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + outage.Down = true; + var before = outage.Attempts; + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(before + 2), + duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + outage.Down = false; + + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + [Fact] public void Readiness_is_NotHosted_after_the_coordinator_stops() { @@ -146,6 +245,18 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes return id; } + private static void SeedNodeState( + IDbContextFactory dbFactory, DeploymentId deploymentId, string nodeId, NodeDeploymentStatus status) + { + using var db = dbFactory.CreateDbContext(); + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = nodeId, DeploymentId = deploymentId.Value, Status = status, + AppliedAtUtc = status == NodeDeploymentStatus.Applying ? null : DateTime.UtcNow, + }); + db.SaveChanges(); + } + private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) { using var db = dbFactory.CreateDbContext(); From f775625805b8a859aafefae5ad6996b1add45f41 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:40:06 -0500 Subject: [PATCH 17/28] =?UTF-8?q?fix(opcua):=20#855=20review=202=20?= =?UTF-8?q?=E2=80=94=20frozen=20replay=20skips=20non-live=20conditions;=20?= =?UTF-8?q?per-hop=20ACL=20resolution;=20drain=20never=20inline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - QueueFrozenReplay's verdict refuses a condition a deploy removed or replaced since the cut (Server-object and folder), so the cut path cannot replay a ghost or judge a replaced scripted alarm on its old folder. - DeliverToNotifierItems resolves the #826 decision's item-independent part (live condition, notifier check, scopes) once per hop; only the grant and SDK role checks stay per item. - A deferred delivery's drain starts on the pool, never inline under Lock. - Test: a condition removed between the cut and its replay (folder + Server object). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- docs/security.md | 3 +- .../OtOpcUaNodeManager.cs | 40 ++++++++++-- .../FolderConditionRefreshWireTests.cs | 63 +++++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/docs/security.md b/docs/security.md index 1d02687b5..a284f90c8 100644 --- a/docs/security.md +++ b/docs/security.md @@ -917,7 +917,8 @@ Behaviours worth knowing before enabling it: event type, source), so a refresh of a folder item could still be followed by older events. Now, at every notifier the node manager owns, the walk serves the attached items itself (`DeliverToNotifierItems`), exactly as it serves Server-object items: the SDK's audit guard, the verdict through `INodeManager3.ValidateEventRolePermissions` (SDK - type/source `ReceiveEvents` + the #826 decision under enforcement) **per event, uncached**, then the item's gate and + type/source `ReceiveEvents` + the #826 decision under enforcement — its item-independent part, the live condition, + the notifier check and the visibility scopes, resolved once per hop) **per event, uncached**, then the item's gate and cut, then `QueueEvent`. The `MonitoredNode2` stays for item attachment and for what still uses its channel (a condition's SDK method audit events, data-change items on the folder), so `InvalidateFolderEventVerdicts` stays. - **No notifier of ours defers any more.** `QueueEvent` never waits for a consumer (the item queue discards oldest at diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index 56e116e51..74104d3d6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -1539,13 +1539,19 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 return; } + DeferredConditionReports queue; lock (_eventReportGate) { - var queue = new DeferredConditionReports(); + queue = new DeferredConditionReports(); _deferredReports[condition] = queue; CountDeferred(+1, condition); - _ = DrainDeferredAsync(condition, queue, delivery.AsTask()); } + + // #855 review 2: started on the pool, never inline. The suspended delivery may complete between the check above and + // here; an inline drain would then run (and end the episode, and clear cuts over every item gate) on this thread, + // under Lock. The queue is registered first, so this condition's later reports wait behind it either way. + var inFlight = delivery.AsTask(); + _ = Task.Run(() => DrainDeferredAsync(condition, queue, inFlight)); } /// Queues a later event behind a condition's deferred one, coalescing past the budget. Under the gate. @@ -1719,13 +1725,33 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 private void DeliverToNotifierItems(MonitoredNode2 monitored, IFilterTarget e) { var deferred = e is ConditionEventSnapshot { Deferred: true } d ? d : null; + + // #855 review 2: the #826 decision's item-independent part — the live condition, whether this notifier is still a + // legitimate notifier of it, and its visibility scopes — is resolved ONCE per hop, not per item (it is what + // IsNotifierEventVisibleTo resolves for an item attached here). Only the grant check and the SDK's role check + // remain per item. + var enforced = AclEnforced; + List? scopes = null; + if (enforced) + { + if (EventConditionOf(e) is not { } named + || !_alarmConditions.TryGetValue(MapKey(named.NodeId), out var live) + || !IsLiveNotifierOf(live, monitored.Node)) + { + return; // fail closed for every item on this notifier + } + + scopes = ConditionVisibilityScopes(live); + } + foreach (var item in monitored.EventMonitoredItems.Values) { if (item is null) continue; try { if (IsAuditEvent(e) && !IsAuditDeliverableTo(item)) continue; - if (ServiceResult.IsBad(((INodeManager3)this).ValidateEventRolePermissions(item, e))) continue; + if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue; + if (scopes is not null && !IsConditionVisibleTo(item, scopes)) continue; QueueConditionEventToItem(item, e, deferred, DeferredFolderDeliveryProbe); } catch (Exception ex) @@ -6701,6 +6727,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 generation, (item, e) => { + // #855 review 2: never replay a condition a deploy has removed or replaced since the cut — its frozen + // state would be a ghost (and, with the ACL on, a replaced scripted alarm would be judged on its old folder). + if (ConditionOf(e) is { } frozen && !IsLiveCondition(frozen)) return false; if (enforced) { if (ConditionOf(e) is not { } condition) return false; @@ -6981,7 +7010,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 DirectRefreshProbe?.Invoke(true, Monitor.IsEntered(Lock)); // #855 review: re-decided per entry right before queueing — a deploy may have revoked since the verdicts. - QueueFrozenReplay(items, states, perItem, generation, (item, e) => IsDirectRefreshEventAllowed(item, e, enforced)); + QueueFrozenReplay( + items, states, perItem, generation, + (item, e) => (ConditionOf(e) is not AlarmConditionState frozen || IsLiveCondition(frozen)) + && IsDirectRefreshEventAllowed(item, e, enforced)); } finally { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs index 0a9bc98b9..59287d809 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs @@ -426,6 +426,69 @@ public sealed class FolderConditionRefreshWireTests after.ShouldBeEmpty("and nothing reaches the item after RefreshEnd either"); } + /// + /// #855 review 2: a condition removed by a deploy between the cut and the queueing of its frozen replay must not be + /// replayed — its frozen Retain=true state would be a ghost no Retain=false ever follows. ACL off, so nothing but the + /// liveness check stands in the way. + /// + /// Refresh a Server-object item instead of a folder item. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_condition_removed_between_the_cut_and_its_replay_is_not_replayed(bool serverObject) + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855RemoveCut", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + // A second subscriber keeps EqA a notifier fan-out the stall applies to. + using var other = await rig.ConnectAsync(anonymous: true, ct); + var otherEvents = new RecordedEvents(); + var otherSub = await RecordedEvents.SubscribeAsync(other, otherEvents, ct); + RecordedEvents.AddEventItem(otherSub, ServerEventRig.UnsNodeId(other, EqA)); + await otherSub.ApplyChangesAsync(ct); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA)); + await subscription.ApplyChangesAsync(ct); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed")); + + var removed = 0; + void Remove(bool cut, bool _) + { + if (cut && Interlocked.Exchange(ref removed, 1) == 0) + rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue(); + } + + if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove; + else rig.Nm.DirectRefreshProbe = Remove; + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0); + await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType)); + stall.Release(); + } + + rig.Nm.ServerObjectRefreshProbe = null; + rig.Nm.DirectRefreshProbe = null; + Volatile.Read(ref removed).ShouldBe(1, "the refresh took the cut path"); + await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + + var (replayed, after) = Window(events.Of(item)); + replayed.ShouldBeEmpty("the condition was removed before its frozen replay was queued"); + after.ShouldBeEmpty("and none of its deferred events follows"); + } + /// /// #855 review: a delivery that fails inline (here: a hop that throws) is over, not suspended — it must not be booked /// as a deferral episode (which moved the episode a concurrent plain refresh checks, and cleared cuts under the From 9060ff55617530747703f611f98f52a9751e8c10 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:46:15 -0500 Subject: [PATCH 18/28] fix(galaxy): #865 review round 2 - wait for the replay's pump, never race a second stream Round 1 replaced a stale-session pump from any advise path and disposed the old one in the background; that could start a second stream while the old one still held the gateway's single-subscriber slot, and a kept current-session pump could be dead with no fault re-reported. - EnsureEventPumpStarted reuses again; only ReplayAsync restarts the pump. RestartEventPumpForReplay keeps a current-session pump only while its loop is running (EventPump.HasEnded), and flags the restart so no advise path starts a competing pump in the dispose gap. - AwaitCurrentEventStreamAsync (bounded by StreamRequestWait) starts a pump only when none exists, WAITS for the replay while the pump is stale or restarting, waits for StreamRequested, and returns the session it verified. SubscribeAsync (per #861 attempt), read-via-subscribe and the probe hook issue and fence the advise under that session, so a reconnect after the check is caught as a straddle and retried. - Read: a deadline (not caller) cancel during the wait falls through to per-ref results. - Tests: reopen-window probe test now asserts the advise waits for the replay's single new stream; first Subscribe/Read wait for the stream; the fencing tests that advise in the window use a short wait bound; the production-path seed test uses an empty galaxy (a real probe advise now starts a stream against the fake endpoint). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 210 +++++++++++------- .../Health/PerPlatformProbeWatcher.cs | 17 +- .../Runtime/EventPump.cs | 3 + .../Runtime/EarlyEventWindowTests.cs | 4 + .../Runtime/GalaxyDriverFixRound1Tests.cs | 5 +- .../Runtime/ProbeInitialValueTests.cs | 64 ++++-- 6 files changed, 199 insertions(+), 104 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 97a2d32cb..7dbb6506d 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -1056,7 +1056,7 @@ public sealed class GalaxyDriver // #865: every re-advise below pushes its initial value once, to whatever stream is attached. EventPump? replayPump; lock (_pumpLock) replayPump = _eventPump; - if (replayPump is not null) await AwaitStreamRequestedAsync(replayPump, cancellationToken).ConfigureAwait(false); + if (replayPump is not null) await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false); // Only subscriptions still holding the previous session's handles: one registered after the // reopen is already live on the recreated session (re-subscribing it would double-advise). @@ -1312,31 +1312,45 @@ public sealed class GalaxyDriver { old = _eventPump; if (old is null) return; // pump never started — nothing to restart - // #865: a pump created after the reopen's fence (an advise in the reopen → replay window - // started it via EnsureEventPumpStarted) already streams the recreated session, and advises - // registered under it may be waiting for their initial values on it — keep it. - if (old.Session == _subscriptions.CurrentSession) return; + // #865: a pump first started after the reopen's fence (no pump existed before the reconnect) + // already streams the recreated session, and advises registered under it may be waiting for + // their initial values on it — keep it, unless its loop has ended (a dead pump is always replaced, + // #859 P-1: a stream fault during Replaying is not re-reported by the supervisor). + if (old.Session == _subscriptions.CurrentSession && !old.HasEnded) return; _eventPump = null; + // #865: until the replacement exists, AwaitCurrentEventStreamAsync must not start one of its + // own — two streams would race for the gateway's single-subscriber slot while the old one detaches. + _pumpRestarting = true; } - // Detach + dispose the faulted pump outside the lock so a slow shutdown doesn't - // block a concurrent EnsureEventPumpStarted; the old loop already exited on fault. - old.OnDataChange -= OnPumpDataChange; - try { old.DisposeAsync().AsTask().GetAwaiter().GetResult(); } - catch (Exception ex) + try { - _logger.LogWarning(ex, - "GalaxyDriver {InstanceId} faulted EventPump dispose failed during replay — continuing.", - _driverInstanceId); - } + // Detach + dispose the faulted pump outside the lock so a slow shutdown doesn't + // block a concurrent EnsureEventPumpStarted; the old loop already exited on fault. + old.OnDataChange -= OnPumpDataChange; + try { old.DisposeAsync().AsTask().GetAwaiter().GetResult(); } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} faulted EventPump dispose failed during replay — continuing.", + _driverInstanceId); + } - // EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the - // recreate if the driver is being disposed or its runtime torn down — the teardown - // already released the pump and a fresh one here would leak past it. - if (_disposed || _runtimeStopping) return; - EnsureEventPumpStarted(); + // EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the + // recreate if the driver is being disposed or its runtime torn down — the teardown + // already released the pump and a fresh one here would leak past it. + if (_disposed || _runtimeStopping) return; + EnsureEventPumpStarted(); + } + finally + { + lock (_pumpLock) _pumpRestarting = false; + } } + /// True while is between dropping the old pump and starting its replacement. + private bool _pumpRestarting; + private void OnSupervisorStateChanged(object? sender, StateTransition transition) { // Reflect supervisor state in DriverHealth + transport forwarder. @@ -1689,10 +1703,19 @@ public sealed class GalaxyDriver var readSubscriber = _subscriber!; // A live pump must exist (values arrive through it); the read listens on ReadDataChange, not // on this instance, so it survives a reconnect's pump restart. - var readPump = EnsureEventPumpStarted(); - // #865: a read waits ONLY for the initial value, which reaches only an already-requested stream. - // Not the caller's token: a cancel here must still yield per-ref Bad snapshots, and the wait is bounded. - await AwaitStreamRequestedAsync(readPump, CancellationToken.None).ConfigureAwait(false); + EnsureEventPumpStarted(); + // #865: a read waits ONLY for the initial value, which reaches only a current-session stream that is + // already requested. The internal deadline bounds this too; a deadline (not caller) cancel falls + // through so the read still answers per-ref, as it does for a deadline during the RPC. + int issueSession; + try + { + issueSession = await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!callerToken.IsCancellationRequested) + { + issueSession = _subscriptions.CurrentSession; + } var subscriptionId = _subscriptions.NextSubscriptionId(); // Pre-allocate one TaskCompletionSource per full-reference so the OnDataChange @@ -1729,7 +1752,7 @@ public sealed class GalaxyDriver // window before the RPC and close it right after registration (the finally below), not after the // value wait: a window held for the whole wait would park every unrelated unknown-handle event. var earlyEventWindow = _subscriptions.BeginPendingSubscribe(); - var session = _subscriptions.CurrentSession; // #861 — the session this SubscribeBulk is issued on + var session = issueSession; // #861/#865 — the session this SubscribeBulk is issued on (its stream verified) try { // The RPC gets only the CALLER's token: the internal deadline bounds our wait, not the RPC, so a slow @@ -1938,8 +1961,6 @@ public sealed class GalaxyDriver var generation = Volatile.Read(ref _runtimeGeneration); var subscriber = _subscriber; var pump = EnsureEventPumpStarted(); - // #865: the advise's initial value reaches only an already-requested stream (no replay). - await AwaitStreamRequestedAsync(pump, cancellationToken).ConfigureAwait(false); var subscriptionId = _subscriptions.NextSubscriptionId(); if (fullReferences.Count == 0) @@ -1972,6 +1993,10 @@ public sealed class GalaxyDriver for (var attempt = 1; ; attempt++) { + // #865: per attempt (a #861 retry is a new session): the advise's initial value reaches only a + // current-session stream that is already requested — the gateway does not replay a fresh stream. + var issueSession = await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false); + // #859: MXAccess's initial value for a new handle can reach the event pump before this // reply is registered (the gateway gives no reply-before-event ordering, and the reply and // the event ride separate RPCs). Open the early-event window BEFORE the RPC so such events @@ -1980,7 +2005,7 @@ public sealed class GalaxyDriver var earlyEventWindow = _subscriptions.BeginPendingSubscribe(); try { - var session = _subscriptions.CurrentSession; // #861 — the session this SubscribeBulk is issued on + var session = issueSession; // #861/#865 — the session this SubscribeBulk is issued on (its stream verified) var results = await subscriber .SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken) .ConfigureAwait(false); @@ -2094,18 +2119,8 @@ public sealed class GalaxyDriver /// reverse map, not by spinning a pump per /// subscription. /// - /// - /// #865: a pump created for an OLDER registry session is replaced, not reused. Between - /// 's session swap and 's pump restart the - /// current pump still streams the dead gateway session; an advise made in that window goes to the - /// recreated session, so its initial value would reach no stream — and, registered under the - /// current session, the replay would not re-advise it either. A pump whose session is current - /// was created after the reopen's second fence, so its stream is on the recreated session. - /// private EventPump EnsureEventPumpStarted() { - EventPump? stale = null; - EventPump pump; lock (_pumpLock) { // Checked BEFORE the reuse: the flag is set before the teardown takes the pump under this @@ -2117,12 +2132,7 @@ public sealed class GalaxyDriver throw new InvalidOperationException( "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync."); } - if (_eventPump is not null && _eventPump.Session == _subscriptions.CurrentSession) return _eventPump; - if (_eventPump is not null) - { - stale = _eventPump; - stale.OnDataChange -= OnPumpDataChange; - } + if (_eventPump is not null) return _eventPump; _eventPump = new EventPump( _subscriber!, _subscriptions, _logger, channelCapacity: _options.MxAccess.EventPumpChannelCapacity, @@ -2131,27 +2141,13 @@ public sealed class GalaxyDriver onReplayGap: OnEventPumpReplayGap); _eventPump.OnDataChange += OnPumpDataChange; _eventPump.Start(); - pump = _eventPump; + return _eventPump; } - - if (stale is not null) DisposeStalePumpInBackground(stale); - return pump; } - /// Dispose a replaced pump off the caller's path (its stream is dead; the dispose only joins its loops). - private void DisposeStalePumpInBackground(EventPump stale) => _ = Task.Run(async () => - { - try { await stale.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) - { - _logger.LogWarning(ex, "GalaxyDriver {InstanceId} stale-session EventPump dispose failed.", _driverInstanceId); - } - }); - /// - /// for infrastructure callers (the probe watcher) that - /// must not fail because the runtime is going away: null — and no pump — while a teardown is in - /// progress or there is no subscriber. + /// for infrastructure callers that must not fail because the + /// runtime is going away: null — and no pump — while a teardown is in progress or there is no subscriber. /// /// The running (or just started) pump; null when none may be started. private EventPump? TryEnsureEventPumpStarted() @@ -2172,43 +2168,95 @@ public sealed class GalaxyDriver } } - /// Upper bound on how long an advise waits for the pump's StreamEvents call to be started. + /// Upper bound on how long an advise waits for a current-session event stream (#865). internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(5); /// - /// #865 — wait, bounded, until has started its StreamEvents call, so an - /// advise made next has a stream for its initial value. A fresh stream (after_worker_sequence = 0) - /// is never replayed, so an initial value pushed before it attaches is lost — and a slow-changing - /// attribute (ScanState, a setpoint) may not publish again for a long time. On timeout the caller - /// proceeds (logged): a late stream is no worse than the pre-#865 behaviour. + /// #865 — make sure an advise about to be issued has an event stream for its initial value, and return + /// the registry session to issue (and register) it under. /// - private async Task AwaitStreamRequestedAsync(EventPump pump, CancellationToken cancellationToken) + /// + /// + /// A fresh StreamEvents (after_worker_sequence = 0) is never replayed, so an initial value pushed before + /// the stream attaches is lost — and a slow-changing attribute (ScanState, a setpoint) may not publish + /// again for a long time. So: start the pump if there is none, and wait until its stream call has started + /// ( — this narrows, it cannot close, the attach race). + /// + /// + /// A pump bound to an OLDER session belongs to a gateway session a reconnect replaced (the reopen → + /// replay window): an advise now would go to the recreated session with no stream for it, and — + /// registered under the current session — the replay would not re-advise it. The pump is not replaced + /// here (only restarts it, so two streams never race for the single-subscriber + /// slot); the advise WAITS for the replay's pump instead. The returned session is the one the pump was + /// verified against: a reconnect after this check makes the caller's registration fail as a session + /// straddle (#861), whose retry comes back through here. + /// + /// + /// Bounded by ; on timeout the caller proceeds with the current session + /// (logged) — no worse than the pre-#865 behaviour. + /// + /// + /// Cancels the wait. + /// The registry session to issue the advise under. + private async Task AwaitCurrentEventStreamAsync(CancellationToken cancellationToken) { - try - { - await pump.StreamRequested.WaitAsync(StreamRequestWait, cancellationToken).ConfigureAwait(false); - } - catch (TimeoutException) + var elapsed = System.Diagnostics.Stopwatch.StartNew(); + while (true) { + var session = _subscriptions.CurrentSession; + EventPump? pump; + bool restarting; + lock (_pumpLock) + { + restarting = _pumpRestarting; + pump = restarting ? null : TryEnsureEventPumpStarted(); + } + if (pump is null && !restarting) return session; // tearing down — the caller's own fences decide + + var remaining = StreamRequestWait - elapsed.Elapsed; + if (pump is not null && pump.Session == session) + { + if (remaining > TimeSpan.Zero) + { + try + { + await pump.StreamRequested.WaitAsync(remaining, cancellationToken).ConfigureAwait(false); + return session; + } + catch (TimeoutException) + { + // fall through to the warning + } + } + else if (pump.StreamRequested.IsCompleted) + { + return session; + } + } + else if (remaining > TimeSpan.Zero) + { + // Reopen → replay window: the replay restarts the pump for the current session. + await Task.Delay(TimeSpan.FromMilliseconds(20), cancellationToken).ConfigureAwait(false); + continue; + } + _logger.LogWarning( - "GalaxyDriver {InstanceId} event stream was not started within {Wait} — advising anyway; initial values may be missed.", + "GalaxyDriver {InstanceId} had no started event stream for the current gateway session within {Wait} — advising anyway; initial values may be missed.", _driverInstanceId, StreamRequestWait); + return _subscriptions.CurrentSession; } } /// - /// #865 — the probe watcher's pre-subscribe hook: start the pump (if this runtime may) and wait for - /// its stream (). A probe's initial ScanState value is usually - /// its only event, and a driver with no authored tags never Reads/Subscribes, so without this its - /// pump — and every host status — never existed. + /// #865 — the probe watcher's pre-subscribe hook (). A probe's + /// initial ScanState value is usually its only event, and a driver with no authored tags never + /// Reads/Subscribes, so without this its pump — and every host status — never existed. /// - private async Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken) + private Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken) { // A seed pass a teardown already cancelled must not start a pump after the teardown finished. cancellationToken.ThrowIfCancellationRequested(); - var pump = TryEnsureEventPumpStarted(); - if (pump is null) return; - await AwaitStreamRequestedAsync(pump, cancellationToken).ConfigureAwait(false); + return AwaitCurrentEventStreamAsync(cancellationToken); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs index 08d2dd395..fc84e59b1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs @@ -47,7 +47,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable // #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the // gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events, // and ScanState rarely changes — a lost initial value leaves the host missing indefinitely). - private readonly Func? _ensureEventStream; + private readonly Func>? _ensureEventStream; // Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe; // we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call. @@ -86,8 +86,9 @@ public sealed class PerPlatformProbeWatcher : IDisposable /// Buffered update interval in milliseconds; must be >= 0. /// /// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries - /// the initial ScanState value is running and requested (the driver starts its - /// and waits for its StreamEvents call to be issued). + /// the initial ScanState value is running and requested for the current gateway session (the + /// driver starts its and waits for its StreamEvents call). Returns the + /// registry session the stream was verified against; the advise is issued and fenced under it. /// internal PerPlatformProbeWatcher( IGalaxySubscriber subscriber, @@ -95,7 +96,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable SubscriptionRegistry? registry, ILogger? logger = null, int bufferedUpdateIntervalMs = 0, - Func? ensureEventStream = null) + Func>? ensureEventStream = null) { _subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber)); _aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator)); @@ -210,12 +211,16 @@ public sealed class PerPlatformProbeWatcher : IDisposable // #865: the stream must be live BEFORE the advise — the initial value is the probe's only // event until the platform's scan state changes, and an unattached stream never sees it. - if (_ensureEventStream is not null) await _ensureEventStream(cancellationToken).ConfigureAwait(false); + // The returned session is the one the stream was verified for: a reconnect after the check makes + // the session test below refuse the reply as a straddle, and the retry comes back through here. + int? streamSession = _ensureEventStream is null + ? null + : await _ensureEventStream(cancellationToken).ConfigureAwait(false); // #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until // PublishBindingsLocked registers the handles below (window closes at method exit). using var earlyEventWindow = _registry?.BeginPendingSubscribe(); - var session = CurrentSession; // the session this SubscribeBulk is issued on + var session = streamSession ?? CurrentSession; // the session this SubscribeBulk is issued on // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState // changes are rare so a coarser interval is usually fine; deployments that need diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index e56d8d57a..dc8905f85 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -154,6 +154,9 @@ internal sealed class EventPump : IAsyncDisposable /// internal int Session => _session; + /// True once the stream loop has exited (faulted, completed by the gateway, or disposed). + internal bool HasEnded => _loop is { IsCompleted: true }; + private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously); /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs index 1d0ade328..474841437 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs @@ -844,6 +844,7 @@ public sealed class EarlyEventWindowTests "g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber) { RecreateOverrideForTests = _ => Task.CompletedTask, + StreamRequestWait = TimeSpan.FromMilliseconds(100), // advises in the reopen window on purpose (#865 bound) }; var delivered = new ConcurrentQueue<(string Ref, double Value)>(); driver.OnDataChange += (_, e) => delivered.Enqueue((e.FullReference, (double)e.Snapshot.Value!)); @@ -1027,6 +1028,9 @@ public sealed class EarlyEventWindowTests "g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber) { RecreateOverrideForTests = _ => Task.CompletedTask, + // These tests advise in the reopen → replay window on purpose (fencing, not values). #865 makes + // such an advise wait for the replay's pump, bounded; keep the bound short so they don't idle. + StreamRequestWait = TimeSpan.FromMilliseconds(100), }; await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); return driver; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs index e2339a751..11c5a095a 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs @@ -272,7 +272,10 @@ public sealed class GalaxyDriverFixRound1Tests [Fact] public async Task Production_path_queries_the_platform_source_at_init() { - var platforms = new ScriptedPlatformSource(); + // An empty galaxy: since #865 a probe advise starts the event stream first, and this owned runtime's + // gRPC channel points at no real gateway — its stream would fault and (correctly) degrade health. + // The query itself is what this test pins. + var platforms = new ScriptedPlatformSource { Script = _ => [] }; await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms) { SessionOpenOverrideForTests = _ => Task.CompletedTask, diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs index 81c986ab2..774aa762e 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs @@ -195,10 +195,10 @@ public sealed class ProbeInitialValueTests /// /// Review finding: a probe advised in the reopen → replay window (the session was swapped, the old - /// pump still streams the dead session) must still get its initial value — the stale-session pump - /// is replaced before the advise, and the replay keeps the replacement (it already streams the - /// recreated session). Reusing the old pump bound the value to the old session, so it never routed, - /// and the replay skipped the platform (its handle is current-session). + /// pump still streams the dead session) must still get its initial value. The advise waits for the + /// replay's restarted pump instead of advising onto the stale one (its value would route under the + /// old session and be lost, and the replay skips a current-session handle) or starting a competing + /// stream (the gateway's default single-subscriber mode refuses a second attach). /// [Fact] public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value() @@ -218,23 +218,55 @@ public sealed class ProbeInitialValueTests await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet sub.ScanStateRunning = false; - await driver.ProbeWatcherForTests.ShouldNotBeNull() + var callsBeforeWindow = sub.CallsSnapshot().Length; + // The window advise must WAIT for the replay's current-session pump rather than advise onto the + // stale one (or start a competing stream). + var sync = driver.ProbeWatcherForTests.ShouldNotBeNull() .SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None); - await TestWait.UntilAsync( - () => StateOf(driver, "LatePlatform") == HostState.Stopped, - "a probe advised in the reopen window must get its initial value", - () => Describe(driver, sub)); - var windowPump = driver.EventPumpInstanceForTests; - windowPump.ShouldNotBeSameAs(oldPump, "the stale-session pump must have been replaced before the advise"); - await driver.InvokeReplayForTestAsync(CancellationToken.None); - driver.EventPumpInstanceForTests.ShouldBeSameAs(windowPump, "the replay keeps a pump already on the current session"); + await TestWait.ForAsync(sync, "the window advise completes once the replay restarted the pump"); + await TestWait.UntilAsync( - () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped), - "the replay re-advises the stale probes and their initial values arrive on the kept pump", + () => StateOf(driver, "LatePlatform") == HostState.Stopped + && Platforms.All(p => StateOf(driver, p) == HostState.Stopped), + "the window advise and the replay's re-advises all get their initial values", () => Describe(driver, sub)); - StateOf(driver, "LatePlatform").ShouldBe(HostState.Stopped); + driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the replay restarted the stale pump"); + var windowCalls = sub.CallsSnapshot()[callsBeforeWindow..]; + var lateAdvise = Array.FindIndex(windowCalls, c => c.Contains("LatePlatform", StringComparison.Ordinal)); + var newStream = Array.IndexOf(windowCalls, "stream"); + newStream.ShouldBeGreaterThanOrEqualTo(0, $"the replay must start a new stream; calls=[{string.Join(" | ", windowCalls)}]"); + lateAdvise.ShouldBeGreaterThan(newStream, $"the window advise must follow the new stream; calls=[{string.Join(" | ", windowCalls)}]"); + windowCalls.Count(c => c == "stream").ShouldBe(1, "no competing stream may be started for the window advise"); + sub.DroppedInitialValues.ShouldBe(0); + } + + /// + /// The data paths too: a first SubscribeAsync and a first ReadAsync on a driver whose pump was never + /// started request the stream BEFORE advising, so their initial values arrive (a read of a value that + /// never changes again would otherwise time out). + /// + [Fact] + public async Task First_subscribe_and_read_advise_only_after_the_stream_is_requested() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) + { + ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10), + }; + await driver.InitializeAsync("{}", CancellationToken.None); + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + driver.OnDataChange += (_, e) => delivered.TrySetResult(e); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + var calls = sub.CallsSnapshot(); + Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the advise; calls=[{string.Join(" | ", calls)}]"); + (await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub))) + .FullReference.ShouldBe("Tank01.Level"); + + var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None); + read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)"); sub.DroppedInitialValues.ShouldBe(0); } From 7783fadae1b8bb54a58c73a9933ea36eabc71c22 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:51:03 -0500 Subject: [PATCH 19/28] =?UTF-8?q?fix(opcua):=20#855=20review=203=20?= =?UTF-8?q?=E2=80=94=20plain=20refresh=20skips=20removed=20conditions;=20p?= =?UTF-8?q?er-hop=20failure=20isolation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Both plain ConditionRefresh paths (Server object and folder) skip a condition a deploy removed or replaced between the off-Lock walk and the queueing (the SDK base did not either; only the cut path checked). - A failure delivering at one notifier hop is logged and the walk continues to the remaining hops; a failure deciding the Server-object items withholds only them. - Test: a condition removed during a plain refresh (folder + Server object). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../OtOpcUaNodeManager.cs | 31 ++++++++++++- .../FolderConditionRefreshWireTests.cs | 43 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index 74104d3d6..f6af07d1a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -1651,7 +1651,19 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 } // Re-checked: a stall may have outlived the condition (the liveness check above ran before it). - if (IsLiveCondition(condition)) DeliverToNotifierItems(monitored, e); + if (IsLiveCondition(condition)) + { + // #855 review 3: one hop's failure must not cost the hops after it (before #855 this hop was a channel + // write, which cannot throw; the SDK isolated failures per item). + try + { + DeliverToNotifierItems(monitored, e); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "OtOpcUaNodeManager: failed to deliver a condition event at notifier {NodeId}", node.NodeId); + } + } } else { @@ -5815,7 +5827,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 // through its OLD equipment folder (#808). if (!IsLiveCondition(node)) return; - var scopes = AclEnforced ? ConditionVisibilityScopes(node) : null; + List? scopes; + try + { + scopes = AclEnforced ? ConditionVisibilityScopes(node) : null; + } + catch (Exception ex) + { + // #855 review 3: withheld from the Server-object items (fail closed), but the walk goes on to the folders. + _logger.LogWarning(ex, "OtOpcUaNodeManager: failed to decide a condition event for Server-object items ({NodeId})", node.NodeId); + return; + } + var reportingSession = (context as ISessionSystemContext)?.SessionId; var deferred = e is ConditionEventSnapshot { Deferred: true } d ? d : null; @@ -6595,6 +6618,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 foreach (var (e, _, scopes) in replay) { + // #855 review 3: never replay a condition a deploy removed or replaced since the walk collected it. + if (ConditionOf(e) is AlarmConditionState walked && !IsLiveCondition(walked)) continue; if (scopes is not null && !IsConditionVisibleTo(item, scopes)) continue; if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue; item.QueueEvent(e); @@ -6878,6 +6903,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 DirectRefreshProbe?.Invoke(false, Monitor.IsEntered(Lock)); foreach (var e in events) { + // #855 review 3: never replay a condition a deploy removed or replaced since the walk collected it. + if (ConditionOf(e) is AlarmConditionState walked && !IsLiveCondition(walked)) continue; if (!IsDirectRefreshEventAllowed(item, e, enforced)) continue; item.QueueEvent(e); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs index 59287d809..b6deba5e2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs @@ -489,6 +489,49 @@ public sealed class FolderConditionRefreshWireTests after.ShouldBeEmpty("and none of its deferred events follows"); } + /// + /// #855 review 3: the plain refresh path (nothing deferred — the only path production takes) collects live + /// conditions off Lock and queues them afterwards; a condition removed in between must not be replayed. + /// + /// Refresh a Server-object item instead of a folder item. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_condition_removed_during_a_plain_refresh_is_not_replayed(bool serverObject) + { + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync("Refresh855RemovePlain", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA)); + await subscription.ApplyChangesAsync(ct); + + rig.Project(native: true, Active("seed")); + await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed")); + + var removed = 0; + void Remove(bool cut, bool _) + { + if (!cut && Interlocked.Exchange(ref removed, 1) == 0) + rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue(); + } + + if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove; + else rig.Nm.DirectRefreshProbe = Remove; + await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); + await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType)); + rig.Nm.ServerObjectRefreshProbe = null; + rig.Nm.DirectRefreshProbe = null; + + Volatile.Read(ref removed).ShouldBe(1, "the plain path ran"); + Window(events.Of(item)).Replayed.ShouldBeEmpty("the condition was removed before the replay was queued"); + } + /// /// #855 review: a delivery that fails inline (here: a hop that throws) is over, not suspended — it must not be booked /// as a deferral episode (which moved the episode a concurrent plain refresh checks, and cleared cuts under the From 494dcb5b87ba8a04c63e4533f0595083575ea325 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 12:54:30 -0500 Subject: [PATCH 20/28] =?UTF-8?q?fix(controlplane,runtime):=20#868=20secon?= =?UTF-8?q?d=20review=20=E2=80=94=20final-outcome=20node=20rows,=20unpersi?= =?UTF-8?q?sted=20acks=20survive=20deadline=20and=20restart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1 DriverHostActor wrote its own NodeDeploymentState row Applied/null as soon as the revision advanced, up to ~105 s before the address-space outcome. The coordinator's recovery counts any non-Applying row as an ack, so a coordinator restart in that window sealed CLEAN while the node was about to report degraded (#601). The row now stays Applying until ReleasePendingAck (or the no-publish-actor path) writes the final Applied, clean or with the degraded reason. A node restart in the window replays the orphan Applying row; /fleet shows Applying until the outcome (docs/Redundancy.md). M2 Acks whose persist failed are tracked in _unpersisted (newest per node). HandleDeadline persists that backlog first and defers its decision while any remain, so a node that acked is never timed out as silent; PreRestart re-sends them to Self so a restart (timers lost) does not drop them. L3 A successful persist cancels the node's retry timer and drops its older pending ack; a retry only fires for the node's current pending ack. L4 A dispatch queued during Loading is sent with a fresh deadline however long the outage lasted (never closed "Dispatch was lost"); the inline re-drive of a genuinely lost dispatch arms the REMAINING deadline. L5 A recovered row with a blank RevisionHash (re-dispatch) or blank NodeId is closed PartiallyFailed with a reason instead of failing every load. L6 Readiness carries PendingAckPersists + LastPersistError; the config-publish health check is Degraded while Ready with a backlog. L7 Ack-persist retries back off (shared Backoff); stack trace on the first failure only. L8 Terminal writes (Sealed / PartiallyFailed / TimedOut / failure reason) only apply to a row still Dispatching/AwaitingApplyAcks. Tests (all fail on the previous code): ControlPlane DbOutage — deadline does not time out an unpersisted ack; unpersisted ack survives a restart; stale retry does not overwrite a newer ack; queued dispatch through an outage longer than the deadline is sent; malformed row closed and coordinator carries on; readiness reports the persist backlog. Runtime DegradedAck — own row stays Applying until the outcome. Host — Ready + backlog is Degraded. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- docs/Redundancy.md | 5 + .../Coordinators/ConfigPublishCoordinator.cs | 199 ++++++++++++---- .../ConfigPublishCoordinatorReadiness.cs | 21 +- .../ConfigPublishCoordinatorHealthCheck.cs | 6 + .../Drivers/DriverHostActor.cs | 20 +- .../ConfigPublishCoordinatorDbOutageTests.cs | 219 +++++++++++++++++- ...onfigPublishCoordinatorHealthCheckTests.cs | 15 ++ .../DriverHostActorDegradedAckTests.cs | 43 +++- 8 files changed, 475 insertions(+), 53 deletions(-) diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 71f229959..1349ee036 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -676,6 +676,11 @@ deployment green fleet-wide. Now: no second rebuild), reporting clean only if that pass is clean. - **A node with a ConfigDb also writes the reason on its own row** (`ApplyAck.DegradedReasonPrefix`), because central only records it when it *accepts* the ack — a timed-out deployment's late ack is discarded. + **That row stays `Applying` until the outcome is known** (lmxopcua#868): it used to be written `Applied`/clean + the moment the revision advanced, and `ConfigPublishCoordinator`'s recovery counts any non-`Applying` row as + an ack, so a coordinator restart inside the ~105 s outcome window sealed the deployment clean while the node + was about to report degraded. `/fleet` therefore shows the node *Applying* until its outcome lands; a restart + of the node in that window replays the orphan `Applying` row (re-applies) instead of restoring it. - **Mixed-version fleets.** An old node never sets the flag, so its acks are treated exactly as before (clean). A new node acking an old central: the old coordinator ignores the unknown field and seals as it always did — no worse than today. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index c2b6770e1..3e0cb2c17 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -98,9 +98,17 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private readonly ConfigPublishCoordinatorReadiness? _readiness; private int _failedLoads; - /// Deployments whose dispatch was queued while Loading (only to word the recovery log correctly). + /// Deployments whose dispatch was queued while Loading (never sent: recovery re-dispatches them). private readonly HashSet _queuedDispatches = new(); + /// + /// Acks of the current deployment that arrived but could not be persisted yet (review M2), one per node — the + /// newest wins. They are retried on a backoff, block the deadline from deciding without them, and are + /// re-sent to Self across a restart (). + /// + private readonly Dictionary _unpersisted = new(NodeIdComparer); + private string? _lastPersistError; + /// /// Where central→node commands are handed off (per-cluster mesh Phase 2), or /// to publish on the mediator directly. @@ -230,7 +238,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU { Receive(HandleDispatch); Receive(HandleAck); - Receive(m => HandleAck(m.Ack)); + Receive(HandleRetryAckPersist); Receive(HandleDeadline); Receive(_ => { /* DPS subscribe confirmation */ }); // A late load outcome / retry tick cannot normally arrive once Ready; ignored, never stashed. @@ -252,6 +260,15 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU StartRecoveryLoad(); } + /// + protected override void PreRestart(Exception reason, object message) + { + // Timers die with this incarnation; the mailbox survives. Re-send every ack that is still unpersisted so the + // next incarnation (which stashes while Loading) persists and counts it — its node will not resend (M2). + foreach (var pending in _unpersisted.Values) Self.Tell(pending.Ack); + base.PreRestart(reason, message); + } + /// protected override void PostStop() { @@ -302,8 +319,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU { _failedLoads++; // Exponential backoff, capped: min * 2^(n-1), with the shift bounded so it never overflows. - var factor = 1L << Math.Min(_failedLoads - 1, 20); - var delay = TimeSpan.FromTicks(Math.Min(_loadRetryMax.Ticks, _loadRetryMin.Ticks * factor)); + var delay = Backoff(_failedLoads); _readiness?.MarkLoading(_failedLoads, msg.Error.Message); _log.Warning(msg.Error, "ConfigPublishCoordinator could not load its recovery state from the ConfigDb (attempt {Attempt}); " + @@ -329,11 +345,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU catch (Exception ex) { _log.Error(ex, "ConfigPublishCoordinator failed to apply its recovered state; retrying the load"); - Timers.Cancel(DeadlineTimerKey); - _current = null; - _expectedAcks.Clear(); - _acks.Clear(); - _degraded.Clear(); + ResetForNext(); HandleRecoveryLoadFailed(new RecoveryLoadFailed(ex)); return; } @@ -359,6 +371,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU _current = new DeploymentId(inflight.DeploymentId); var nodeStates = inflight.NodeStates; + // Review L5: a row recovery cannot interpret (blank revision on a row it must re-dispatch, blank node id) would + // throw on every load and keep the coordinator Loading forever. Close it with a reason and carry on. + var malformed = DescribeMalformed(inflight); + if (malformed is not null) + { + _log.Error("Coordinator cannot recover deployment {Id}: {Problem}. Closing it as PartiallyFailed", _current, malformed); + using var db = _dbFactory.CreateDbContext(); + UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.PartiallyFailed, onlyIfInFlight: true); + SetFailureReason(db, Truncate( + $"The deploy coordinator could not recover this deployment: {malformed}. Redeploy.")); + db.SaveChanges(); + ResetForNext(); + return; + } + // Resume the deadline timer using the remaining time. The deadline runs from when the // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough // proxy — we don't track a separate "dispatched at" column). @@ -377,21 +404,27 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that // keeps throwing ends up on the other branch once the deadline passes); past it the row is // closed as TimedOut with a reason that names the condition. - if (remaining > TimeSpan.Zero) + if (_queuedDispatches.Contains(_current.Value)) { - if (_queuedDispatches.Contains(_current.Value)) - _log.Info( - "Coordinator dispatching deployment {Id}, queued while its recovery state was loading", - _current); - else - _log.Warning( - "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + - "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + - "re-dispatching with {Remaining} of the apply deadline left", - _current, remaining); + // Review L4: its dispatch was QUEUED here while the load was failing — it was never sent, so however + // long the outage lasted it is not a lost dispatch. Send it now with a full, fresh deadline. + _log.Info( + "Coordinator dispatching deployment {Id}, queued while its recovery state was loading", + _current); HandleDispatch(new DispatchDeployment( _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); } + else if (remaining > TimeSpan.Zero) + { + _log.Warning( + "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + + "re-dispatching with {Remaining} of the apply deadline left", + _current, remaining); + // The REMAINING time (review L4): CreatedAtUtc never moves, so a re-drive must not restart the clock. + HandleDispatch(new DispatchDeployment( + _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()), remaining); + } else { // Deliberately INLINE, unlike the populated-set deadline below: with no expected set no queued ack @@ -413,7 +446,8 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU } // Every expected ack is already in the DB (a seal whose SaveChanges failed, or a fused node's own - // DriverHostActor row): finish it now. Waiting for the deadline would leave it in flight — the deadline + // DriverHostActor row — which that actor writes Applied only once the address-space outcome is known, so a + // non-Applying row is always a FINAL outcome; review M1): finish it now. Waiting for the deadline would leave it in flight — the deadline // can only time out an INCOMPLETE set. if (_acks.Count == _expectedAcks.Count) { @@ -434,7 +468,11 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU _current, _acks.Count, _expectedAcks.Count); } - private void HandleDispatch(DispatchDeployment msg) + private void HandleDispatch(DispatchDeployment msg) => HandleDispatch(msg, deadline: null); + + /// The dispatch. + /// The apply deadline to arm; null = the full . + private void HandleDispatch(DispatchDeployment msg, TimeSpan? deadline) { // A duplicate of the deployment already in flight with a seeded expected set (a dispatch queued while the // recovery load ran, for the very row that load re-drove) must not reset the acks already gathered. @@ -529,7 +567,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU // directly, so this actor no longer knows which transport is in force. if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg)); else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg)); - Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), _applyDeadline); + Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), deadline ?? _applyDeadline); if (_expectedAcks.Count == 0) { @@ -567,26 +605,68 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU // Persist FIRST: the row is the system of record (driver-only nodes never write their own, and nodes do not // resend). A throw here used to restart the actor and drop the ack; it is re-queued instead. + if (!TryPersistAck(msg)) return; + if (_acks.Count < _expectedAcks.Count) return; + CompleteFromAcks(); + } + + /// + /// Persists an ack and, only on success, counts it (and drops any older unpersisted ack of that node, with its + /// retry timer — review L3). On failure the ack is held in and retried on a backoff. + /// + /// when the ack is persisted and counted. + private bool TryPersistAck(ApplyAck msg) + { try { PersistNodeAck(msg); } catch (Exception ex) { - _log.Warning(ex, - "Could not persist the ApplyAck from {Node} for {Id}; retrying in {Delay}", - msg.NodeId, msg.DeploymentId, _loadRetryMin); - Timers.StartSingleTimer(AckRetryTimerKey(msg), new RetryAckPersist(msg), _loadRetryMin); - return; + var attempts = _unpersisted.TryGetValue(msg.NodeId, out var prior) ? prior.Attempts + 1 : 1; + _unpersisted[msg.NodeId] = new PendingPersist(msg, attempts); + _lastPersistError = ex.Message; + var delay = Backoff(attempts); + // Review L7: the stack trace once, then one line per retry. + if (attempts == 1) + _log.Warning(ex, "Could not persist the ApplyAck from {Node} for {Id}; retrying in {Delay}", + msg.NodeId, msg.DeploymentId, delay); + else + _log.Warning("Still cannot persist the ApplyAck from {Node} for {Id} (attempt {Attempt}: {Error}); retrying in {Delay}", + msg.NodeId, msg.DeploymentId, attempts, ex.Message, delay); + Timers.StartSingleTimer(AckRetryTimerKey(msg.NodeId), new RetryAckPersist(msg), delay); + ReportPersistBacklog(); + return false; + } + + if (_unpersisted.Remove(msg.NodeId)) + { + Timers.Cancel(AckRetryTimerKey(msg.NodeId)); + if (_unpersisted.Count == 0) _lastPersistError = null; + ReportPersistBacklog(); } _acks[msg.NodeId] = msg.Outcome; // An old node never sets Degraded, so its ack takes the clean branch exactly as before. if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId); else _degraded.Remove(msg.NodeId); + return true; + } - if (_acks.Count < _expectedAcks.Count) return; - CompleteFromAcks(); + /// A retry tick: only the node's CURRENT unpersisted ack is retried; a superseded one is dropped (L3). + private void HandleRetryAckPersist(RetryAckPersist msg) + { + if (!_unpersisted.TryGetValue(msg.Ack.NodeId, out var pending) || !ReferenceEquals(pending.Ack, msg.Ack)) return; + HandleAck(msg.Ack); + } + + private void ReportPersistBacklog() => _readiness?.MarkAckPersistBacklog(_unpersisted.Count, _lastPersistError); + + /// min * 2^(attempt-1), capped at the max — shared by the load and ack-persist retries. + private TimeSpan Backoff(int attempt) + { + var factor = 1L << Math.Min(Math.Max(attempt, 1) - 1, 20); + return TimeSpan.FromTicks(Math.Min(_loadRetryMax.Ticks, _loadRetryMin.Ticks * factor)); } /// Seals (all Applied) or marks partially failed — called once every expected ack is in. @@ -598,7 +678,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU MarkPartiallyFailed(); } - private static string AckRetryTimerKey(ApplyAck msg) => $"ack-retry:{msg.DeploymentId.Value:N}:{msg.NodeId.Value}"; + private static string AckRetryTimerKey(NodeId node) => $"ack-retry:{node.Value.ToUpperInvariant()}"; private void PersistNodeAck(ApplyAck msg) { @@ -634,7 +714,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU { if (_current is null) return; using var db = _dbFactory.CreateDbContext(); - UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.Sealed, sealNow: true); + UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.Sealed, sealNow: true, onlyIfInFlight: true); if (_degraded.Count > 0) { // Still Sealed: every node DID apply this revision and is running it, and "latest sealed" is @@ -663,7 +743,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU { if (_current is null) return; using var db = _dbFactory.CreateDbContext(); - UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.PartiallyFailed); + UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.PartiallyFailed, onlyIfInFlight: true); db.SaveChanges(); _log.Warning("Deployment {Id} partially failed; acks={Acks}", _current.Value, string.Join(",", _acks.Select(kv => $"{kv.Key.Value}={kv.Value}"))); @@ -678,6 +758,22 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU msg.DeploymentId, _current); return; } + // Review M2: an ack that ARRIVED but is not yet persisted must never be timed out as "no ack from" its node. + // Persist the backlog now; if the DB still refuses, defer the decision (the terminal write needs it too). + if (_unpersisted.Count > 0) + { + foreach (var pending in _unpersisted.Values.ToList()) TryPersistAck(pending.Ack); + if (_unpersisted.Count > 0) + { + var delay = Backoff(_unpersisted.Values.Min(p => p.Attempts)); + _log.Warning( + "Deployment {Id} reached its apply deadline with {Count} received ack(s) not yet persisted; deferring " + + "the outcome {Delay} until they are", _current.Value, _unpersisted.Count, delay); + Timers.StartSingleTimer(DeadlineTimerKey, msg, delay); + return; + } + } + // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped. if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count) @@ -689,7 +785,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU } using var db = _dbFactory.CreateDbContext(); - UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); + UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut, onlyIfInFlight: true); if (_expectedAcks.Count == 0) { SetFailureReason(db, Truncate( @@ -745,6 +841,13 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private void ResetForNext() { Timers.Cancel(DeadlineTimerKey); + foreach (var node in _unpersisted.Keys) Timers.Cancel(AckRetryTimerKey(node)); + if (_unpersisted.Count > 0) + { + _unpersisted.Clear(); + _lastPersistError = null; + ReportPersistBacklog(); + } _current = null; _expectedAcks.Clear(); _acks.Clear(); @@ -757,6 +860,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU /// An ack whose persist failed, re-delivered by its retry timer. private sealed record RetryAckPersist(ApplyAck Ack); + /// A received ack not yet persisted, and how many persists of it have failed. + private sealed record PendingPersist(ApplyAck Ack, int Attempts); + + /// Why recovery cannot interpret a row (review L5), or null when it can. + private static string? DescribeMalformed(RecoveredDeployment d) + { + if (d.NodeStates.Count == 0 && string.IsNullOrWhiteSpace(d.RevisionHash)) + return "its RevisionHash is blank, so it cannot be re-dispatched"; + if (d.NodeStates.Any(n => string.IsNullOrWhiteSpace(n.NodeId))) + return "one of its NodeDeploymentState rows has a blank NodeId"; + return null; + } + public sealed record DeadlineElapsed(DeploymentId DeploymentId); /// The recovery read's result: the newest in-flight deployment, or null when none. @@ -784,9 +900,10 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU /// stub stays tracked so can add to the same UPDATE. /// private static void UpdateDeploymentStatus( - OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null) + OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null, + bool onlyIfInFlight = false) { - var entry = TrackStub(db, id, rowVersion); + var entry = TrackStub(db, id, rowVersion, onlyIfInFlight); if (entry is null) return; entry.Entity.Status = status; entry.Property(x => x.Status).IsModified = true; @@ -801,7 +918,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private void SetFailureReason(OtOpcUaConfigDbContext db, string reason) { if (_current is null) return; - var entry = TrackStub(db, _current.Value); + var entry = TrackStub(db, _current.Value, onlyIfInFlight: true); if (entry is null) return; entry.Entity.FailureReason = reason; entry.Property(x => x.FailureReason).IsModified = true; @@ -815,7 +932,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU /// guard); reads the current one. /// private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( - OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null) + OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null, bool onlyIfInFlight = false) { var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); if (tracked is not null) return tracked; @@ -824,9 +941,13 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU var stamp = db.Deployments .AsNoTracking() .Where(x => x.DeploymentId == id.Value) - .Select(x => new { x.RowVersion }) + .Select(x => new { x.RowVersion, x.Status }) .FirstOrDefault(); if (stamp is null) return null; + // Review L8: a terminal outcome is written once. A row something else already closed (AdminOps' lost- + // dispatch closure, a previous incarnation) is left as it is; the RowVersion check covers the race. + if (onlyIfInFlight && stamp.Status is not (DeploymentStatus.Dispatching or DeploymentStatus.AwaitingApplyAcks)) + return null; rowVersion = stamp.RowVersion; } return db.Deployments.Attach(new Deployment diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs index 0074a8eee..f96bab29a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinatorReadiness.cs @@ -18,8 +18,11 @@ public enum ConfigPublishCoordinatorState /// Consecutive failed recovery loads (0 unless ). /// The last load failure's message, or null. /// When the state was entered. +/// Received acks the coordinator could not persist yet and is retrying. +/// The last ack-persist failure's message, or null. public sealed record ConfigPublishCoordinatorSnapshot( - ConfigPublishCoordinatorState State, int FailedLoadAttempts, string? LastError, DateTime SinceUtc); + ConfigPublishCoordinatorState State, int FailedLoadAttempts, string? LastError, DateTime SinceUtc, + int PendingAckPersists = 0, string? LastPersistError = null); /// /// Thread-safe state bridge from the actor to the node's health endpoint @@ -39,11 +42,23 @@ public sealed class ConfigPublishCoordinatorReadiness public void MarkLoading(int failedAttempts, string? lastError) { var since = _current.State == ConfigPublishCoordinatorState.Loading ? _current.SinceUtc : DateTime.UtcNow; - _current = new(ConfigPublishCoordinatorState.Loading, failedAttempts, lastError, since); + _current = _current with + { + State = ConfigPublishCoordinatorState.Loading, FailedLoadAttempts = failedAttempts, LastError = lastError, SinceUtc = since, + }; } + /// The ack-persist backlog changed (review L6): acks received but not yet written, and why. + /// Acks awaiting a persist retry. + /// The last persist failure's message, or null. + public void MarkAckPersistBacklog(int pending, string? lastError) => + _current = _current with { PendingAckPersists = pending, LastPersistError = pending > 0 ? lastError : null }; + /// The recovery state is loaded. - public void MarkReady() => _current = new(ConfigPublishCoordinatorState.Ready, 0, null, DateTime.UtcNow); + public void MarkReady() => _current = _current with + { + State = ConfigPublishCoordinatorState.Ready, FailedLoadAttempts = 0, LastError = null, SinceUtc = DateTime.UtcNow, + }; /// The singleton stopped on this node. public void MarkNotHosted() => _current = new(ConfigPublishCoordinatorState.NotHosted, 0, null, DateTime.UtcNow); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs index 579eb660d..94ff59990 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ConfigPublishCoordinatorHealthCheck.cs @@ -29,9 +29,15 @@ public sealed class ConfigPublishCoordinatorHealthCheck(ConfigPublishCoordinator ["failedLoadAttempts"] = s.FailedLoadAttempts, }; if (s.LastError is not null) data["lastError"] = s.LastError; + data["pendingAckPersists"] = s.PendingAckPersists; + if (s.LastPersistError is not null) data["lastPersistError"] = s.LastPersistError; return Task.FromResult(s.State switch { + ConfigPublishCoordinatorState.Ready when s.PendingAckPersists > 0 => HealthCheckResult.Degraded( + $"Deploy coordinator cannot persist {s.PendingAckPersists} received deploy ack(s) to the ConfigDb " + + $"(last: {s.LastPersistError}); retrying with backoff. The deployment's outcome waits for them.", + data: data), ConfigPublishCoordinatorState.Loading => HealthCheckResult.Degraded( s.FailedLoadAttempts == 0 ? "Deploy coordinator is loading its recovery state from the ConfigDb; deployments are queued until it completes." diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 254a7ee21..56ea2ad3b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -2984,7 +2984,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS } _currentRevision = msg.Revision; - UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null); + // lmxopcua#868 review M1: the own row stays APPLYING (written by BeginDirectApply) until the + // address-space outcome is known — AwaitAddressSpaceOutcomeThenAck / ReleasePendingAck write the + // final Applied (clean, or with the degraded reason). Writing Applied/null here, up to ~105 s before + // that outcome, made the row read as a CLEAN ack: a coordinator restart in that window recovered it + // as complete and sealed the deployment clean while this node was about to report degraded (#601). + // A restart of THIS node in the window now replays the orphan Applying row (re-applies), which is + // the honest answer to "outcome unknown". // The Applied ack is NOT sent here any more. It used to go out right now, before the // address-space apply below had even been dequeued, so a degraded apply (swallowed sink faults, // _lastApplied not advanced) still sealed the deployment green fleet-wide. The ack is now held @@ -3069,6 +3075,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS if (_opcUaPublishActor is null) { _currentRevisionDegraded = false; + UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applied, failureReason: null); SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation); return; } @@ -3126,13 +3133,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS if (_currentRevision is { } cur && cur == pending.Revision) _currentRevisionDegraded = degraded; + // The own row has been Applying since the apply began (lmxopcua#868 review M1); the FINAL outcome is + // written here, clean or degraded. Central adds the reason from the ack only when it ACCEPTS the ack (a + // timed-out or superseded deployment's ack is discarded), so a node with a ConfigDb records it itself; a + // driver-only node no-ops here. + UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied, + degraded ? ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded") : null); if (degraded) { - // The apply wrote this node's own row Applied/null before the outcome was known. Central - // adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack - // is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here. - UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied, - ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded")); OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "degraded")); _log.Warning( "DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it", diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs index ae10d1966..bb3045a67 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs @@ -116,8 +116,9 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes } /// - /// Review finding 1: a recovered deployment whose every expected ack is already in the DB (a seal whose - /// SaveChanges failed, or a fused node's own row) must be completed — inside the deadline it used to wait for + /// Review finding 1: a recovered deployment whose every expected ack is already in the DB as a FINAL outcome + /// (a seal whose SaveChanges failed, or a fused node's own row — which DriverHostActor writes Applied only + /// once the address-space outcome is known; a pre-outcome row stays Applying, second review M1) must be completed — inside the deadline it used to wait for /// acks that had already landed, past it the deadline's "race" guard returned without sealing; either way it /// sat in AwaitingApplyAcks forever and AdminOps refused every later deploy. /// @@ -214,6 +215,214 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); } + /// + /// Second review M2: an ack that arrived but whose persist is still failing at the deadline must not be timed + /// out as "no ack from" its node. The deadline is deferred until the backlog persists, then the deployment seals. + /// + [Fact] + public void Deadline_does_not_time_out_a_received_but_unpersisted_ack() + { + var (direct, outage, deploymentId, coordinator, _) = DispatchedSingleNode(TimeSpan.FromSeconds(2)); + + outage.Down = true; + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + // Hold the DB down past the 2 s apply deadline. + Thread.Sleep(TimeSpan.FromSeconds(3.5)); + outage.Down = false; + + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Second review M2: an unpersisted ack must survive an actor restart (its retry timer does not). Here a second + /// dispatch's guard read throws while the DB is down, restarting the coordinator. + /// + [Fact] + public void Unpersisted_ack_survives_a_coordinator_restart() + { + var (direct, outage, deploymentId, coordinator, _) = DispatchedSingleNode(NoTimeout); + + outage.Down = true; + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + var before = outage.Attempts; + AwaitAssert(() => outage.Attempts.ShouldBeGreaterThan(before), + duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + // Its guard read throws (DB down) => the coordinator restarts. + coordinator.Tell(new DispatchDeployment(DeploymentId.NewId(), TestRevision, CorrelationId.NewId())); + Thread.Sleep(TimeSpan.FromMilliseconds(500)); + outage.Down = false; + + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == deploymentId.Value).Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Second review L3: a retry of a node's OLDER unpersisted ack must not overwrite the newer ack that persisted + /// after it (it used to flip the node to Failed and turn a clean deploy PartiallyFailed). + /// + [Fact] + public void A_stale_ack_retry_does_not_overwrite_a_newer_persisted_ack() + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(direct, Node, "site-a-2:4053"); + var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, NoTimeout, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + // Node 1's first answer (Failed) cannot be persisted; its corrected answer (Applied) can. + outage.FailNext(1); + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed, "first try", CorrelationId.NewId())); + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + Thread.Sleep(TimeSpan.FromMilliseconds(600)); // several retry periods + using (var db = direct.CreateDbContext()) + db.NodeDeploymentStates.Single(s => s.NodeId == Node).Status.ShouldBe(NodeDeploymentStatus.Applied); + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-2:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Second review L4: a dispatch QUEUED while the load failed for longer than the apply deadline was never sent; + /// it must be dispatched (fresh deadline) once the load succeeds, not closed "Dispatch was lost". + /// + [Fact] + public void Dispatch_queued_through_an_outage_longer_than_the_deadline_is_sent_not_closed_as_lost() + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true }; + SeedNodes(direct, Node); + var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow); + var deadline = TimeSpan.FromSeconds(3); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, deadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); + Thread.Sleep(deadline + TimeSpan.FromSeconds(1)); + outage.Down = false; + + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node }); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Second review L5: an in-flight row recovery cannot interpret (blank RevisionHash on a row it must re-dispatch) + /// used to throw on every load and keep the coordinator Loading forever. It is closed with a reason, and the + /// coordinator serves the next deployment. + /// + [Fact] + public void Malformed_in_flight_row_is_closed_with_a_reason_and_the_coordinator_carries_on() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, Node); + var bad = DeploymentId.NewId(); + using (var db = dbFactory.CreateDbContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = bad.Value, RevisionHash = " ", Status = DeploymentStatus.Dispatching, + CreatedBy = "test", CreatedAtUtc = DateTime.UtcNow, + }); + db.SaveChanges(); + } + var readiness = new ConfigPublishCoordinatorReadiness(); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + dbFactory, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + + AwaitAssert(() => + { + readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready); + using var db = dbFactory.CreateDbContext(); + var row = db.Deployments.Single(d => d.DeploymentId == bad.Value); + row.Status.ShouldBe(DeploymentStatus.PartiallyFailed); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("RevisionHash"); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + var next = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(1)); + coordinator.Tell(new DispatchDeployment(next, TestRevision, CorrelationId.NewId())); + coordinator.Tell(new ApplyAck(next, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == next.Value).Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// Second review L6: an ack-persist backlog is visible on readiness while Ready, and clears. + [Fact] + public void Readiness_reports_the_ack_persist_backlog() + { + var (_, outage, deploymentId, coordinator, readiness) = DispatchedSingleNode(NoTimeout); + + outage.Down = true; + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready); + readiness.Current.PendingAckPersists.ShouldBe(1); + readiness.Current.LastPersistError.ShouldNotBeNull(); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + outage.Down = false; + AwaitAssert(() => + { + readiness.Current.PendingAckPersists.ShouldBe(0); + readiness.Current.LastPersistError.ShouldBeNull(); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// A single-node deployment dispatched and AwaitingApplyAcks through an outage-capable factory. + private (IDbContextFactory Direct, OutageDbFactory Outage, DeploymentId Id, IActorRef Coordinator, + ConfigPublishCoordinatorReadiness Readiness) DispatchedSingleNode(TimeSpan applyDeadline) + { + var dbName = Guid.NewGuid().ToString("N"); + var direct = NewInMemoryDbFactory(dbName); + var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(direct, Node); + var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow); + var readiness = new ConfigPublishCoordinatorReadiness(); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + outage, applyDeadline, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = direct.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + return (direct, outage, deploymentId, coordinator, readiness); + } + [Fact] public void Readiness_is_NotHosted_after_the_coordinator_stops() { @@ -281,15 +490,21 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes { private volatile bool _down; private int _attempts; + private int _failNext; public bool Down { get => _down; set => _down = value; } public int Attempts => Volatile.Read(ref _attempts); + /// The next context creations throw, then the DB is reachable again. + public void FailNext(int count) => Volatile.Write(ref _failNext, count); + public OtOpcUaConfigDbContext CreateDbContext() { Interlocked.Increment(ref _attempts); if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated outage)"); + if (Volatile.Read(ref _failNext) > 0 && Interlocked.Decrement(ref _failNext) >= 0) + throw new InvalidOperationException("ConfigDb unreachable (simulated blip)"); return inner.CreateDbContext(); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs index a9202432c..560028524 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs @@ -36,6 +36,21 @@ public sealed class ConfigPublishCoordinatorHealthCheckTests (await Check(r)).Status.ShouldBe(HealthStatus.Healthy); } + [Fact] + public async Task Ready_with_an_ack_persist_backlog_is_degraded_and_clears() + { + var r = new ConfigPublishCoordinatorReadiness(); + r.MarkReady(); + r.MarkAckPersistBacklog(2, "timeout"); + var result = await Check(r); + result.Status.ShouldBe(HealthStatus.Degraded); + result.Description.ShouldNotBeNull(); + result.Description.ShouldContain("timeout"); + + r.MarkAckPersistBacklog(0, null); + (await Check(r)).Status.ShouldBe(HealthStatus.Healthy); + } + [Fact] public async Task Not_hosted_and_untracked_are_healthy() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs index 7af4f16e1..99f261ee1 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs @@ -174,12 +174,49 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); } + [Fact] + public void The_local_row_stays_Applying_until_the_address_space_outcome_is_known() + { + // lmxopcua#868 second review M1: the node used to write its own row Applied/null the moment the revision + // advanced, up to ~105 s before the outcome. ConfigPublishCoordinator's recovery counts a non-Applying row + // as an ack, so a coordinator restart in that window sealed the deployment CLEAN while the node was about + // to report degraded. The row must carry only the final outcome. + var (db, deploymentId) = SeedOneTagDeployment(); + var coordinator = CreateTestProbe(); + var publish = CreateTestProbe(); + var actor = Sys.ActorOf(DriverHostActor.Props( + db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, + opcUaPublishActor: publish.Ref)); + + actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); + var rebuild = publish.FishForMessage(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + + // The ack is held: the row must still read Applying (the apply's synchronous writes are done by now — the + // rebuild Tell above is issued after them on the actor thread). + using (var ctx = db.CreateDbContext()) + ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value) + .Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applying); + + publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null)); + coordinator.ExpectMsg(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeFalse(); + + AwaitAssert(() => + { + using var ctx = db.CreateDbContext(); + var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value); + row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied); + row.FailureReason.ShouldBeNull(); + row.AppliedAtUtc.ShouldNotBeNull(); + }, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + [Fact] public void A_degraded_apply_records_its_reason_on_the_local_NodeDeploymentState_row() { - // #601 review — a node WITH a ConfigDb writes its own row Applied/null before the outcome is known. - // Central later adds the reason from the ack, but only when it accepts that ack (a stale/timed-out - // deployment's ack is discarded), so the node must not leave its own row reading clean. + // #601 review — a node WITH a ConfigDb records its own row. Central later adds the reason from the ack, + // but only when it accepts that ack (a stale/timed-out deployment's ack is discarded), so the node must + // not leave its own row reading clean. var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var publish = CreateTestProbe(); From 1c1705fa7748fbc4a2e6dc420d0a747054f5e28e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:00:19 -0500 Subject: [PATCH 21/28] fix(galaxy): #865 review round 3 - budget-safe stream wait, dead pump, fast-path drop - StreamRequestWait 1 s (under Tier-A Read 2 s / Subscribe 5 s budgets) and a stale pump is only waited on while the supervisor is reopening or replaying (or a restart is underway); in a reconnect backoff the advise proceeds at once. The no-stream log is a rate-limited Warning while Healthy, Debug otherwise. - A current-session pump whose loop has ended is not a stream: if the supervisor already reports Healthy (its fault was swallowed while it was recovering) the failure is re-reported to start a new cycle. - Subscribe/Read no longer start the pump directly (only AwaitCurrentEventStreamAsync does, honouring the replay restart flag); the teardown refusal is an explicit ThrowIfRuntimeStopping. - SubscriptionRegistry.ResolveOrPark fast path: the handle lookup and the open-window check were not atomic, so a subscribe that registered the handle and closed its window in between got its initial value dropped as stale (found by a stress run: the first event of a bulk advise lost). It re-looks-up before declaring stale. Deterministic test via a seam. - Tests: first Read on a fresh driver, replay of a data subscription. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 120 +++++++++++++++--- .../Runtime/SubscriptionRegistry.cs | 19 ++- .../Runtime/EarlyEventWindowTests.cs | 25 ++++ .../Runtime/ProbeInitialValueTests.cs | 45 +++++++ 4 files changed, 189 insertions(+), 20 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 7dbb6506d..c9dae1420 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -1703,7 +1703,7 @@ public sealed class GalaxyDriver var readSubscriber = _subscriber!; // A live pump must exist (values arrive through it); the read listens on ReadDataChange, not // on this instance, so it survives a reconnect's pump restart. - EnsureEventPumpStarted(); + ThrowIfRuntimeStopping(); // the pump itself is started by AwaitCurrentEventStreamAsync below // #865: a read waits ONLY for the initial value, which reaches only a current-session stream that is // already requested. The internal deadline bounds this too; a deadline (not caller) cancel falls // through so the read still answers per-ref, as it does for a deadline during the RPC. @@ -1960,7 +1960,7 @@ public sealed class GalaxyDriver // generation, so the Register below can tell its SubscribeBulk straddled a teardown. var generation = Volatile.Read(ref _runtimeGeneration); var subscriber = _subscriber; - var pump = EnsureEventPumpStarted(); + ThrowIfRuntimeStopping(); // the pump is started per attempt by AwaitCurrentEventStreamAsync (#865) var subscriptionId = _subscriptions.NextSubscriptionId(); if (fullReferences.Count == 0) @@ -2014,7 +2014,6 @@ public sealed class GalaxyDriver switch (TryRegisterForGeneration(subscriptionId, bindings, generation, session)) { case RegisterOutcome.Registered: - _ = pump; // keep the pump alive for the subscription's lifetime return new GalaxySubscriptionHandle(subscriptionId); case RegisterOutcome.TornDown: @@ -2113,11 +2112,25 @@ public sealed class GalaxyDriver } } + /// A Subscribe/Read is refused while a teardown is in progress (the pump would belong to a dying subscriber). + private void ThrowIfRuntimeStopping() + { + lock (_pumpLock) + { + if (_runtimeStopping) + { + throw new InvalidOperationException( + "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync."); + } + } + } + /// - /// Lazily start the on the first subscribe. The pump is - /// shared across every subscription on this driver — fan-out happens through the - /// reverse map, not by spinning a pump per - /// subscription. + /// Lazily start the on the first advise. Since #865 the advise paths + /// (probe seed, Subscribe, Read) go through , and only the + /// replay replaces an existing pump. The pump is shared across every subscription on this driver — + /// fan-out happens through the reverse map, not by spinning a pump + /// per subscription. /// private EventPump EnsureEventPumpStarted() { @@ -2168,8 +2181,12 @@ public sealed class GalaxyDriver } } - /// Upper bound on how long an advise waits for a current-session event stream (#865). - internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(5); + /// + /// Upper bound on how long an advise waits for a current-session event stream (#865). Kept well under + /// Galaxy's Tier-A Read (2 s) / Subscribe (5 s) call budgets, so the "proceed anyway" fallback is + /// reachable before the resilience pipeline times the call out. + /// + internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(1); /// /// #865 — make sure an advise about to be issued has an event stream for its initial value, and return @@ -2180,20 +2197,30 @@ public sealed class GalaxyDriver /// A fresh StreamEvents (after_worker_sequence = 0) is never replayed, so an initial value pushed before /// the stream attaches is lost — and a slow-changing attribute (ScanState, a setpoint) may not publish /// again for a long time. So: start the pump if there is none, and wait until its stream call has started - /// ( — this narrows, it cannot close, the attach race). + /// ( — this narrows, it cannot close, the attach race: the gateway + /// offers no attach acknowledgement). /// /// /// A pump bound to an OLDER session belongs to a gateway session a reconnect replaced (the reopen → /// replay window): an advise now would go to the recreated session with no stream for it, and — /// registered under the current session — the replay would not re-advise it. The pump is not replaced - /// here (only restarts it, so two streams never race for the single-subscriber - /// slot); the advise WAITS for the replay's pump instead. The returned session is the one the pump was - /// verified against: a reconnect after this check makes the caller's registration fail as a session - /// straddle (#861), whose retry comes back through here. + /// here (only restarts it, so two streams never race for the gateway's + /// single-subscriber slot); while the supervisor is actually reopening/replaying the advise WAITS for the + /// replay's pump. During a reconnect BACKOFF (TransportLost) no replay is imminent, so it does not wait. + /// The returned session is the one the pump was verified against: a reconnect after this check makes the + /// caller's registration fail as a session straddle (#861), whose retry comes back through here. /// /// - /// Bounded by ; on timeout the caller proceeds with the current session - /// (logged) — no worse than the pre-#865 behaviour. + /// Bounded by per call (a #861 retry starts a new budget). On timeout the + /// caller proceeds with the current session — no worse than the pre-#865 behaviour, but note the cost: + /// such an advise registers as current, so the replay will not re-advise it, and its initial value is + /// lost until the attribute next changes. + /// + /// + /// A current-session pump whose loop has ENDED is not a stream (its + /// completes on exit). If the supervisor already reports Healthy, its fault was swallowed while it was + /// recovering (it ignores faults outside Healthy) — the driver would be deaf while Healthy — so it is + /// reported again to start a new recovery cycle. /// /// /// Cancels the wait. @@ -2216,16 +2243,22 @@ public sealed class GalaxyDriver var remaining = StreamRequestWait - elapsed.Elapsed; if (pump is not null && pump.Session == session) { + if (pump.HasEnded) + { + ReportEndedPump(pump); + return session; + } if (remaining > TimeSpan.Zero) { try { await pump.StreamRequested.WaitAsync(remaining, cancellationToken).ConfigureAwait(false); + if (pump.HasEnded) ReportEndedPump(pump); return session; } catch (TimeoutException) { - // fall through to the warning + // fall through to the log } } else if (pump.StreamRequested.IsCompleted) @@ -2233,18 +2266,67 @@ public sealed class GalaxyDriver return session; } } - else if (remaining > TimeSpan.Zero) + else if (remaining > TimeSpan.Zero && ReplayExpectedSoon()) { // Reopen → replay window: the replay restarts the pump for the current session. await Task.Delay(TimeSpan.FromMilliseconds(20), cancellationToken).ConfigureAwait(false); continue; } + LogNoCurrentStream(); + return _subscriptions.CurrentSession; + } + } + + /// + /// True when a stale pump will be replaced shortly: the supervisor is reopening/replaying, or there is + /// no supervisor (the injected-seam path, where the replay is driven directly), or a restart is underway. + /// + private bool ReplayExpectedSoon() + { + if (Volatile.Read(ref _pumpRestarting)) return true; + var supervisor = _supervisor; + if (supervisor is null) return true; + var state = supervisor.CurrentState; + return state is ReconnectSupervisor.State.Reopening or ReconnectSupervisor.State.Replaying; + } + + /// A current-session pump's loop ended while the supervisor was not listening (see AwaitCurrentEventStreamAsync). + private void ReportEndedPump(EventPump pump) + { + var supervisor = _supervisor; + if (supervisor is null || _runtimeStopping || supervisor.CurrentState != ReconnectSupervisor.State.Healthy) return; + lock (_pumpLock) + { + if (!ReferenceEquals(_eventPump, pump)) return; // already replaced + } + _logger.LogWarning( + "GalaxyDriver {InstanceId} event stream has ended while the reconnect supervisor reports Healthy — re-reporting the transport failure.", + _driverInstanceId); + try { supervisor.ReportTransportFailure(new InvalidOperationException("The Galaxy event stream ended; no events can be received.")); } + catch (ObjectDisposedException) { /* driver disposing */ } + } + + private long _lastNoStreamWarningTicks; + + /// Warning at most once a minute while the supervisor is Healthy; Debug otherwise (an outage would flood). + private void LogNoCurrentStream() + { + var supervisor = _supervisor; + var now = Environment.TickCount64; + var last = Interlocked.Read(ref _lastNoStreamWarningTicks); + if ((supervisor is null || supervisor.CurrentState == ReconnectSupervisor.State.Healthy) + && (last == 0 || now - last >= 60_000) + && Interlocked.CompareExchange(ref _lastNoStreamWarningTicks, now, last) == last) + { _logger.LogWarning( "GalaxyDriver {InstanceId} had no started event stream for the current gateway session within {Wait} — advising anyway; initial values may be missed.", _driverInstanceId, StreamRequestWait); - return _subscriptions.CurrentSession; + return; } + _logger.LogDebug( + "GalaxyDriver {InstanceId} advising without a started current-session event stream (supervisor not Healthy or warning rate-limited).", + _driverInstanceId); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs index d7a76d37c..95e6f83d9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs @@ -314,6 +314,12 @@ internal sealed class SubscriptionRegistry /// internal Action? AfterMapWriteForTests { get; set; } + /// + /// Test seam (#865): invoked on 's lock-free fast path after the handle + /// lookup missed, before the open-window check. + /// + internal Action? AfterFastPathMissForTests { get; set; } + /// /// Test seam (M-1): parks unconditionally, producing the parked list for a /// resolved handle that the atomic registration otherwise makes impossible. @@ -496,7 +502,18 @@ internal sealed class SubscriptionRegistry { subscribers = ResolveSubscribers(ev.ItemHandle); if (subscribers.Count > 0) return EarlyEventOutcome.Resolved; - if (Volatile.Read(ref _pendingSubscribes) == 0) return EarlyEventOutcome.Stale; + AfterFastPathMissForTests?.Invoke(); + if (Volatile.Read(ref _pendingSubscribes) == 0) + { + // #865 (found by its stress run): the lookup above and this read are not atomic. A + // subscribe can register the handle AND close its window in between — then "no window + // open" does not mean "stale", and dropping here lost the initial value for good (the + // first event of a bulk advise, while the dispatch thread was slow). The window closes + // only after its registration, so once no window is open a re-lookup sees every + // registration that window made; nothing is held, so no older event can be overtaken. + subscribers = ResolveSubscribers(ev.ItemHandle); + return subscribers.Count > 0 ? EarlyEventOutcome.Resolved : EarlyEventOutcome.Stale; + } } lock (_parkGate) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs index 474841437..84c62252c 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EarlyEventWindowTests.cs @@ -898,6 +898,31 @@ public sealed class EarlyEventWindowTests driver.SubscriptionsForTests.ResolveSubscribers(77).ShouldBeEmpty(); } + /// + /// #865 stress finding: the dispatch fast path looked the handle up, then checked for an open window — + /// not atomically. A subscribe that registered the handle AND closed its window in between made the + /// event "stale" and its initial value was dropped for good. It must resolve. + /// + [Fact] + public void A_registration_landing_between_the_fast_path_lookup_and_the_window_check_still_resolves() + { + var registry = new SubscriptionRegistry(); + var session = registry.CurrentSession; + var window = registry.BeginPendingSubscribe(); + registry.AfterFastPathMissForTests = () => + { + registry.AfterFastPathMissForTests = null; + registry.TryRegister(registry.NextSubscriptionId(), [new TagBinding("Tank.A", 7)], session).ShouldBeTrue(); + window.Dispose(); // the subscribe's window closes right after its registration + }; + + registry.ResolveOrPark(DataChange(7, 1), session, out var subscribers, out _).ShouldBe(EarlyEventOutcome.Resolved); + subscribers.Single().FullReference.ShouldBe("Tank.A"); + + // Positive control for the Stale branch: an unknown handle with no window open is still dropped. + registry.ResolveOrPark(DataChange(8, 1), session, out _, out _).ShouldBe(EarlyEventOutcome.Stale); + } + /// I-B: a purge that completes after a reconnect does not delete the new session's parked values. [Fact] public void A_purge_for_an_older_session_is_a_no_op() diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs index 774aa762e..36f152888 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs @@ -265,11 +265,56 @@ public sealed class ProbeInitialValueTests (await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub))) .FullReference.ShouldBe("Tank01.Level"); + sub.DroppedInitialValues.ShouldBe(0); + } + + /// A first ReadAsync on a fresh driver (no pump yet) requests the stream before its advise. + [Fact] + public async Task First_read_advises_only_after_the_stream_is_requested() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) + { + ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10), + }; + await driver.InitializeAsync("{}", CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeFalse("precondition: the read is the first advise"); + var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None); + + var calls = sub.CallsSnapshot(); + Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the read's advise; calls=[{string.Join(" | ", calls)}]"); read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)"); sub.DroppedInitialValues.ShouldBe(0); } + /// + /// The replay's re-advise of DATA subscriptions (which has no per-advise hook) waits for the restarted + /// pump's stream, so each re-advised tag's new initial value arrives. The value flips so only a + /// post-replay event can satisfy the wait. + /// + [Fact] + public async Task Reconnect_replay_delivers_re_advised_data_subscriptions_initial_values() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + var last = new System.Collections.Concurrent.ConcurrentDictionary(); + driver.OnDataChange += (_, e) => last[e.FullReference] = e.Snapshot.Value; + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is true, + "positive control: the subscribe's initial value arrives", () => Describe(driver, sub)); + + sub.ScanStateRunning = false; // the fake's next initial value + await driver.InvokeReplayForTestAsync(CancellationToken.None); + + await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is false, + "the replay's re-advise must deliver its new initial value", + () => Describe(driver, sub) + $" last=[{string.Join(",", last.Select(kv => $"{kv.Key}={kv.Value}"))}] parked={driver.ParkedEarlyEventCountForTests} h2=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(2).Select(b => b.FullReference))}] h1=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(1).Select(b => b.FullReference))}]"); + sub.DroppedInitialValues.ShouldBe(0); + } + /// /// A probe sync on a driver whose pump was never started (no platform source, e.g. the /// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just From ec17a8797b58189b142c9edcf4cc0ae21095bdf4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:05:06 -0500 Subject: [PATCH 22/28] =?UTF-8?q?fix(opcua):=20#855=20review=204=20?= =?UTF-8?q?=E2=80=94=20per-item=20"known"=20in=20the=20folder=20cut;=20pin?= =?UTF-8?q?=20the=20post-stall=20liveness=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RefreshDirectItemsWithCut keys its "decided off Lock" set per (item, condition), so a condition one item's walk saw is still decided for another. - The deferral warning no longer blames slow folder subscribers (no folder of ours defers since #855). - FolderEventFloodWireTests' removal test now asserts the folder item got nothing and the Server object exactly the pre-stall event (it passed with the post-stall liveness re-check deleted). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../OtOpcUaNodeManager.cs | 13 +++++++------ .../FolderEventFloodWireTests.cs | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index f6af07d1a..0bed75dad 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -1806,9 +1806,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 { _lastDeferralWarningUtc = now; _logger.LogWarning( - "OtOpcUaNodeManager: condition events are being deferred — an event notifier's delivery channel is " - + "full (first: {NodeId}). Folder event subscribers are not keeping up, or Lock is held across a large " - + "burst; events stay ordered per condition, and past {PerCondition} waiting per condition the oldest " + "OtOpcUaNodeManager: condition events are being deferred — a notifier hop the node manager does not " + + "serve itself suspended (first: {NodeId}); since #855 no folder of ours does, so this is unexpected. " + + "Events stay ordered per condition, and past {PerCondition} waiting per condition the oldest " + "give way to the newest (Gitea #838)", condition.NodeId, MaxDeferredEventsPerCondition); } @@ -6948,7 +6948,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 // Verdicts per (item, condition) before Lock, from a live walk: they depend on the event type, source, notifier // wiring and grants, not on the condition's state. var allowed = new HashSet<(IEventMonitoredItem, NodeState)>(); - var known = new HashSet(System.Collections.Generic.ReferenceEqualityComparer.Instance); + // Per (item, condition): a condition one item's walk saw is still undecided for another item (#855 review 4). + var known = new HashSet<(IEventMonitoredItem, NodeState)>(); foreach (var item in items) { if (AttachedNotifierOf(item) is not { } notifier) continue; @@ -6957,7 +6958,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 foreach (var e in events) { if (ConditionOf(e) is not { } node) continue; - known.Add(node); + known.Add((item, node)); if (IsDirectRefreshEventAllowed(item, e, enforced)) allowed.Add((item, node)); } } @@ -7015,7 +7016,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3 // A condition materialised since the verdicts were taken is decided here (rare). if (!allowed.Contains((item, instance))) { - if (known.Contains(instance)) continue; + if (known.Contains((item, instance))) continue; if (!IsDirectRefreshEventAllowed(item, target, enforced)) continue; } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs index 97f2271de..266637e48 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs @@ -193,7 +193,10 @@ public sealed class FolderEventFloodWireTests Flooded(onServer).ShouldNotBeEmpty(); // None of the deferred tail — which was still waiting when the condition was removed — reached anyone. Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber"); - Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber"); + // Only the in-flight f0 ever reached the stalled EqA hop, and the condition was gone when the stall released: the + // post-stall liveness re-check must have withheld it (#855 review 4 — "< FloodSize - 1" passed without that check). + Flooded(onA).ShouldBeEmpty("no ghost reached the folder subscriber"); + Flooded(onServer).ShouldBe([0], "only the event reported before the stall reached the Server object"); } private static AlarmConditionSnapshot State(string message) => From 8653960ea3c841787606947d4475e9f751eeb443 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:07:26 -0500 Subject: [PATCH 23/28] fix(galaxy): #865 review round 4 - replay fails on a dead stream, backoff-aware wait - ReplayAsync fails when its restarted pump's stream has already ended, so the supervisor backs off and stays Degraded instead of going Healthy over a dead stream (whose next advise re-report restarted at attempt 1 with no backoff - a tight reconnect loop). - ReconnectSupervisor.AttemptInProgress (reopen through replay); advises wait for a stale pump's replacement only while an attempt is running, not for a whole backoff (the state still reads Reopening/Replaying there). - ReconnectSupervisor: _disposed is set under the state lock and re-checked inside ReportTransportFailure, so a late report cannot start a recovery loop that StopAsync already missed. - EventPump.HasEnded is a flag set before StreamRequested is released, and a stream that ends/faults on its first MoveNext is marked ended first. - SubscriptionRegistry fast path: a miss now falls through to the locked path (lookup + session + window check atomically) instead of a lock-free re-lookup. - Tests: replay over an immediately-ending stream fails; AttemptInProgress across attempt/backoff; ReportTransportFailure after stop; the reopen-window test uses a generous wait bound. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 21 ++++++-- .../Runtime/EventPump.cs | 21 +++++++- .../Runtime/ReconnectSupervisor.cs | 36 +++++++++++-- .../Runtime/SubscriptionRegistry.cs | 16 ++---- .../Runtime/ProbeInitialValueTests.cs | 34 +++++++++++++ .../Runtime/ReconnectSupervisorTests.cs | 50 +++++++++++++++++++ 6 files changed, 156 insertions(+), 22 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index c9dae1420..8800aa65f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -1056,7 +1056,19 @@ public sealed class GalaxyDriver // #865: every re-advise below pushes its initial value once, to whatever stream is attached. EventPump? replayPump; lock (_pumpLock) replayPump = _eventPump; - if (replayPump is not null) await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false); + if (replayPump is not null) + { + await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false); + // A replay whose fresh pump already died must FAIL, so the supervisor backs off and stays + // Degraded — succeeding would go Healthy with a dead stream, and the next advise's re-report + // would start a new cycle at attempt 1 (no backoff): a tight reconnect loop. + lock (_pumpLock) replayPump = _eventPump; + if (replayPump is { HasEnded: true }) + { + throw new InvalidOperationException( + "The Galaxy event stream ended during the reconnect replay — retrying after backoff."); + } + } // Only subscriptions still holding the previous session's handles: one registered after the // reopen is already live on the recreated session (re-subscribing it would double-advise). @@ -2279,7 +2291,7 @@ public sealed class GalaxyDriver } /// - /// True when a stale pump will be replaced shortly: the supervisor is reopening/replaying, or there is + /// True when a stale pump will be replaced shortly: a reopen/replay attempt is running, or there is /// no supervisor (the injected-seam path, where the replay is driven directly), or a restart is underway. /// private bool ReplayExpectedSoon() @@ -2287,8 +2299,9 @@ public sealed class GalaxyDriver if (Volatile.Read(ref _pumpRestarting)) return true; var supervisor = _supervisor; if (supervisor is null) return true; - var state = supervisor.CurrentState; - return state is ReconnectSupervisor.State.Reopening or ReconnectSupervisor.State.Replaying; + // Not the state: between attempts (the backoff delay, up to MaxBackoff) it still reads + // Reopening/Replaying while no replay is coming any time soon. + return supervisor.AttemptInProgress; } /// A current-session pump's loop ended while the supervisor was not listening (see AwaitCurrentEventStreamAsync). diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index dc8905f85..cdb36a264 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -155,7 +155,11 @@ internal sealed class EventPump : IAsyncDisposable internal int Session => _session; /// True once the stream loop has exited (faulted, completed by the gateway, or disposed). - internal bool HasEnded => _loop is { IsCompleted: true }; + internal bool HasEnded => Volatile.Read(ref _ended); + + // Set at the very start of the stream loop's finally — before StreamRequested is completed there — so a + // waiter released by a dying loop always sees it ended (the loop task itself completes only later). + private bool _ended; private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -212,7 +216,19 @@ internal sealed class EventPump : IAsyncDisposable // Start the first MoveNext BEFORE awaiting it: that synchronous step issues the RPC, which is // the moment StreamRequested reports (#865). var next = enumerator.MoveNextAsync(); - _streamRequested.TrySetResult(); + if (next.IsCompleted) + { + // Already decided (an item, an immediate end, or a fault): consume it now, so a stream that + // ended or faulted at once is seen as ended (finally) BEFORE StreamRequested releases anyone. + var hasFirst = await next.ConfigureAwait(false); + if (!hasFirst) Volatile.Write(ref _ended, true); + _streamRequested.TrySetResult(); + next = new ValueTask(hasFirst); + } + else + { + _streamRequested.TrySetResult(); + } for (; await next.ConfigureAwait(false); next = enumerator.MoveNextAsync()) { var ev = enumerator.Current; @@ -274,6 +290,7 @@ internal sealed class EventPump : IAsyncDisposable } finally { + Volatile.Write(ref _ended, true); // A loop that died before issuing the RPC must not strand a StreamRequested waiter. _streamRequested.TrySetResult(); if (enumerator is not null) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs index 7b6c64101..41c1e5a52 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs @@ -104,6 +104,15 @@ public sealed class ReconnectSupervisor : IDisposable get { lock (_stateLock) return _state; } } + private bool _attemptInProgress; + + /// + /// True while a reopen or replay is actually running — false between attempts (the backoff delay, + /// during which the state still reads Reopening/Replaying) and while Healthy. #865: an advise waits + /// for the replay's pump only while one is imminent. + /// + public bool AttemptInProgress => Volatile.Read(ref _attemptInProgress); + /// True when CurrentState != Healthy. Drivers map this to DriverState.Degraded. public bool IsDegraded { @@ -136,6 +145,9 @@ public sealed class ReconnectSupervisor : IDisposable lock (_stateLock) { + // Re-checked under the lock (StopAsync sets it under the same lock): a report that passed the + // check above while a stop ran must not start a loop nothing would ever cancel. + ObjectDisposedException.ThrowIf(_disposed, this); _lastError = cause.Message; if (_state != State.Healthy) { @@ -208,13 +220,21 @@ public sealed class ReconnectSupervisor : IDisposable // === Reopening phase === lock (_stateLock) TransitionLocked(State.Reopening, _lastError); + // In progress from the reopen through the replay (no gap between the phases); cleared on any + // exit from the attempt — failure (then the backoff), cancellation or success. + Volatile.Write(ref _attemptInProgress, true); try { await _reopen(ct).ConfigureAwait(false); } - catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + Volatile.Write(ref _attemptInProgress, false); + return; + } catch (Exception ex) { + Volatile.Write(ref _attemptInProgress, false); _logger.LogWarning(ex, "Galaxy reopen failed (attempt {Attempt})", attempt); lock (_stateLock) { _lastError = ex.Message; } continue; // back to backoff + retry @@ -234,6 +254,10 @@ public sealed class ReconnectSupervisor : IDisposable lock (_stateLock) { _lastError = ex.Message; } continue; // back to backoff + retry } + finally + { + Volatile.Write(ref _attemptInProgress, false); + } // === Done === lock (_stateLock) @@ -282,12 +306,14 @@ public sealed class ReconnectSupervisor : IDisposable /// A task that completes once the recovery loop has exited. public async Task StopAsync() { - if (_disposed) return; - _disposed = true; - CancellationTokenSource? cts; Task? loop; - lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; } + lock (_stateLock) + { + if (_disposed) return; + _disposed = true; + cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; + } // The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS // throws on Cancel, and a disposed one is a no-op on Dispose. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs index 95e6f83d9..5f4aaf82f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs @@ -503,17 +503,11 @@ internal sealed class SubscriptionRegistry subscribers = ResolveSubscribers(ev.ItemHandle); if (subscribers.Count > 0) return EarlyEventOutcome.Resolved; AfterFastPathMissForTests?.Invoke(); - if (Volatile.Read(ref _pendingSubscribes) == 0) - { - // #865 (found by its stress run): the lookup above and this read are not atomic. A - // subscribe can register the handle AND close its window in between — then "no window - // open" does not mean "stale", and dropping here lost the initial value for good (the - // first event of a bulk advise, while the dispatch thread was slow). The window closes - // only after its registration, so once no window is open a re-lookup sees every - // registration that window made; nothing is held, so no older event can be overtaken. - subscribers = ResolveSubscribers(ev.ItemHandle); - return subscribers.Count > 0 ? EarlyEventOutcome.Resolved : EarlyEventOutcome.Stale; - } + // #865 (found by its stress run): a miss is NOT decided here. The lookup above and an + // open-window check are not atomic — a subscribe can register the handle AND close its window + // in between, and declaring the event stale then lost its initial value for good (the first + // event of a bulk advise, while the dispatch thread was slow). Misses are rare; the locked path + // below re-does the lookup, the session check and the window check atomically. } lock (_parkGate) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs index 36f152888..7eabd47ab 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs @@ -51,6 +51,11 @@ public sealed class ProbeInitialValueTests /// Initial values pushed while no stream was attached — lost, as on the real gateway. public int DroppedInitialValues; + private int _streams; + + /// Streams beyond this count end immediately after attaching (a gateway refusing/ending them). + public int StreamsThatStayOpen = int.MaxValue; + /// The ScanState value the next advise reports. public volatile bool ScanStateRunning = true; @@ -90,6 +95,11 @@ public sealed class ProbeInitialValueTests Calls.Add("stream"); _attached = mine; } + if (Interlocked.Increment(ref _streams) > StreamsThatStayOpen) + { + lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; } + yield break; // the gateway completed this stream at once + } try { await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) @@ -208,6 +218,9 @@ public sealed class ProbeInitialValueTests platformSource: new FixedPlatformSource(Platforms)) { RecreateOverrideForTests = _ => Task.CompletedTask, + // Generous: the window advise must still be waiting when the replay below starts the pump, even + // on a stalled agent; the wait ends the moment that pump exists. + StreamRequestWait = TimeSpan.FromSeconds(20), }; await driver.InitializeAsync("{}", CancellationToken.None); await TestWait.UntilAsync( @@ -315,6 +328,27 @@ public sealed class ProbeInitialValueTests sub.DroppedInitialValues.ShouldBe(0); } + /// + /// A replay whose restarted pump's stream ends at once must FAIL (the supervisor then backs off and + /// stays Degraded) rather than succeed onto a dead stream — succeeding went Healthy, and the next + /// advise's re-report started a new cycle with no backoff: a tight reconnect loop. Positive control: + /// the first replay, whose stream stays open, succeeds. + /// + [Fact] + public async Task Replay_fails_when_the_restarted_stream_ends_at_once() + { + var sub = new GatewayLikeSubscriber { StreamsThatStayOpen = 2 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // stream 1 + + await driver.InvokeReplayForTestAsync(CancellationToken.None); // stream 2 stays open — succeeds + + await Should.ThrowAsync( + () => driver.InvokeReplayForTestAsync(CancellationToken.None), // stream 3 ends at once + "a replay must not report success over a dead event stream"); + } + /// /// A probe sync on a driver whose pump was never started (no platform source, e.g. the /// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs index accc819cf..b99f3b43c 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs @@ -15,6 +15,56 @@ public sealed class ReconnectSupervisorTests new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20)); + /// + /// #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff + /// between attempts — when the state still reads Reopening. Advises wait for a replay only while one is + /// actually running, not for a whole backoff. + /// + [Fact] + public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff() + { + var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reopens = 0; + using var sup = new ReconnectSupervisor( + reopen: async _ => + { + if (Interlocked.Increment(ref reopens) == 1) + { + reopenEntered.TrySetResult(); + await releaseReopen.Task; + throw new IOException("gateway still down"); + } + }, + replay: _ => Task.CompletedTask, + backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff + + sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt"); + sup.ReportTransportFailure(new IOException("drop")); + await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start"); + sup.AttemptInProgress.ShouldBeTrue("a reopen is running"); + + releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min + await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag"); + sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists"); + } + + /// #865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE). + [Fact] + public async Task ReportTransportFailure_after_stop_never_starts_a_loop() + { + var reopens = 0; + var sup = new ReconnectSupervisor( + reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; }, + replay: _ => Task.CompletedTask, + options: FastBackoff()); + await sup.StopAsync(); + + Should.Throw(() => sup.ReportTransportFailure(new IOException("late"))); + sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy); + Volatile.Read(ref reopens).ShouldBe(0); + } + /// Verifies that the supervisor starts in a healthy state with no errors. [Fact] public void InitialState_IsHealthy() From d80bdf63690d13c7858ac3f7b2c9eb91929e290b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:12:33 -0500 Subject: [PATCH 24/28] fix(galaxy): #865 review round 5 - a failure during the replay fails the attempt ReconnectSupervisor swallowed a transport failure reported while Replaying, so a restarted event stream refused/ended a round trip after the replay began still let the replay go Healthy over a dead stream (and the next advise's re-report restarted at attempt 1 with no backoff). A failure reported during Replaying is now remembered and turns that attempt's success into a retry with backoff. ReplayAsync's synchronous dead-pump throw yields to a teardown's cancellation. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 4 +++ .../Runtime/ReconnectSupervisor.cs | 33 ++++++++++++++++--- .../Runtime/ReconnectSupervisorTests.cs | 29 ++++++++++++++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 8800aa65f..2cfc682e4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -1065,6 +1065,10 @@ public sealed class GalaxyDriver lock (_pumpLock) replayPump = _eventPump; if (replayPump is { HasEnded: true }) { + // A teardown cancelling this replay can end the pump too: that is a clean cancellation. + cancellationToken.ThrowIfCancellationRequested(); + // The synchronous case; a stream that ends a round trip later reports a transport failure + // while Replaying, which the supervisor itself turns into a failed attempt. throw new InvalidOperationException( "The Galaxy event stream ended during the reconnect replay — retrying after backoff."); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs index 41c1e5a52..43ab2d920 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs @@ -106,6 +106,9 @@ public sealed class ReconnectSupervisor : IDisposable private bool _attemptInProgress; + // Set (under _stateLock) by a ReportTransportFailure that arrives while Replaying; cleared on entering Replaying. + private bool _faultDuringReplay; + /// /// True while a reopen or replay is actually running — false between attempts (the backoff delay, /// during which the state still reads Reopening/Replaying) and while Healthy. #865: an advise waits @@ -151,7 +154,10 @@ public sealed class ReconnectSupervisor : IDisposable _lastError = cause.Message; if (_state != State.Healthy) { - // Already recovering — nothing else to do. + // Already recovering — nothing else to do, except: a failure DURING the replay means the + // replay ran onto a transport that is already gone again (#865 — e.g. the restarted event + // stream refused or ended); the replay's success must not then be reported as Healthy. + if (_state == State.Replaying) _faultDuringReplay = true; _logger.LogDebug("Transport failure reported during {State}: {Message}", _state, cause.Message); return; } @@ -241,7 +247,11 @@ public sealed class ReconnectSupervisor : IDisposable } // === Replaying phase === - lock (_stateLock) TransitionLocked(State.Replaying, _lastError); + lock (_stateLock) + { + _faultDuringReplay = false; + TransitionLocked(State.Replaying, _lastError); + } try { @@ -259,11 +269,24 @@ public sealed class ReconnectSupervisor : IDisposable Volatile.Write(ref _attemptInProgress, false); } - // === Done === + // === Done — unless the transport failed again while the replay ran === + bool faultedDuringReplay; lock (_stateLock) { - _lastError = null; - TransitionLocked(State.Healthy, null); + faultedDuringReplay = _faultDuringReplay; + if (!faultedDuringReplay) + { + _lastError = null; + TransitionLocked(State.Healthy, null); + } + } + if (faultedDuringReplay) + { + // Going Healthy here would hide a dead transport (the fault was swallowed above), and the + // next report would restart at attempt 1 with no backoff. Retry this cycle with backoff. + _logger.LogWarning( + "Galaxy replay completed but the transport failed again while it ran (attempt {Attempt}) — retrying", attempt); + continue; } _logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt); return; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs index b99f3b43c..10ecb016a 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs @@ -49,6 +49,35 @@ public sealed class ReconnectSupervisorTests sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists"); } + /// + /// #865 review: a transport failure reported WHILE the replay runs (e.g. the restarted event stream is + /// refused a round trip later) must fail that attempt — not go Healthy over a dead transport. The + /// supervisor retries (with backoff) and goes Healthy only after a clean replay. Without the fix the + /// first replay's success went Healthy and replay ran once. + /// + [Fact] + public async Task A_failure_reported_during_the_replay_fails_that_attempt() + { + ReconnectSupervisor? self = null; + var replays = 0; + using var sup = new ReconnectSupervisor( + reopen: _ => Task.CompletedTask, + replay: _ => + { + if (Interlocked.Increment(ref replays) == 1) + { + self!.ReportTransportFailure(new IOException("stream refused during replay")); + } + return Task.CompletedTask; + }, + options: FastBackoff()); + self = sup; + + sup.ReportTransportFailure(new IOException("drop")); + await TestWait.HealthyAsync(sup, "the second, clean replay must recover"); + Volatile.Read(ref replays).ShouldBe(2, "the faulted replay must not count as recovered"); + } + /// #865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE). [Fact] public async Task ReportTransportFailure_after_stop_never_starts_a_loop() From 780c87eba374b18ff4b6e471c78c6884f17b9d81 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:13:36 -0500 Subject: [PATCH 25/28] =?UTF-8?q?fix(controlplane,runtime):=20#868=20third?= =?UTF-8?q?=20review=20=E2=80=94=20bounded=20failure=20reasons,=20poison?= =?UTF-8?q?=20ack=20writes,=20cache-booted=20revisions=20unconfirmed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 PersistNodeAck wrote a Failed ack's FailureReason unbounded; the column is nvarchar(2048), so on SQL Server the write failed forever, the ack retried forever, the deadline deferred forever and every later deploy was refused. - One shared bound, Configuration.FailureReasons.Truncate (MaxLength 2048), used by the coordinator and DriverHostActor.UpsertNodeDeploymentState (exception messages and degraded reasons). - A non-transient DbUpdateException (not concurrency, no transient DbException / timeout / socket / IO underneath) is POISON: the ack is counted from memory with an Error, never retried. - Deadline deferral on unpersisted acks is capped at 10 x the retry ceiling; past it the outcome is decided from the acks held in memory. 2 A node restarting in the outcome window boots the cached revision (cache is written before the outcome) and never acks. Cache-booted revisions (BootstrapFromCache / TryBootFromCache) are now marked unconfirmed, so a re-dispatch re-applies and acks the real outcome instead of a clean short-circuit. docs/Redundancy.md corrected (replay only on the ConfigDb boot path). 3 A lost dispatch re-driven near its deadline now gets a FULL deadline (a node needs its whole outcome timeout + margin = ApplyDeadline, so any shorter remainder timed out nodes that applied). No dispatched-at column: a second restart recomputes from CreatedAtUtc and closes it instead (documented). 4 docs/Redundancy.md upgrade note: do not deploy while the central pair is mid rolling-upgrade across this change. Tests (all fail on the previous code): oversized Failed reason truncated (on a SaveChanges interceptor that enforces the column like SQL Server); rejected ack write counted from memory; bounded deadline deferral; full deadline on a near-deadline re-drive; cache-booted revision re-applies on re-dispatch; oversized degraded reason truncated on the node's own row. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- docs/Redundancy.md | 14 ++- .../FailureReasons.cs | 20 ++++ .../Coordinators/ConfigPublishCoordinator.cs | 101 ++++++++++++---- .../Drivers/DriverHostActor.cs | 12 +- .../ConfigPublishCoordinatorDbOutageTests.cs | 109 ++++++++++++++++++ .../Harness/ConstraintEnforcingDbFactory.cs | 69 +++++++++++ .../DriverHostActorBootFromCacheTests.cs | 25 ++++ .../DriverHostActorDegradedAckTests.cs | 52 +++++++++ 8 files changed, 378 insertions(+), 24 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/FailureReasons.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 1349ee036..150c15cb5 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -679,8 +679,18 @@ deployment green fleet-wide. Now: **That row stays `Applying` until the outcome is known** (lmxopcua#868): it used to be written `Applied`/clean the moment the revision advanced, and `ConfigPublishCoordinator`'s recovery counts any non-`Applying` row as an ack, so a coordinator restart inside the ~105 s outcome window sealed the deployment clean while the node - was about to report degraded. `/fleet` therefore shows the node *Applying* until its outcome lands; a restart - of the node in that window replays the orphan `Applying` row (re-applies) instead of restoring it. + was about to report degraded. `/fleet` therefore shows the node *Applying* until its outcome lands. + A restart of the node inside that window depends on how it boots: + - a `Direct` node that can read the ConfigDb finds the orphan `Applying` row and **replays** the apply, which + acks the real outcome; + - a node that boots **from its LocalDb cache** (every `FetchAndCache` driver-only node, or a `Direct` node + whose ConfigDb is unreachable) restores the cached revision — the cache is written before the outcome — and + sends **no ack**, so central times that deployment out even though the node runs it. The restored revision + is marked *unconfirmed*, so **re-deploying it re-applies** and acks the real outcome rather than + short-circuiting a clean `Applied`. +- **Upgrade note (lmxopcua#868): do not deploy while the central pair is mid rolling-upgrade across this + change.** An old-build fused central node still writes its own row `Applied` before the outcome, which a + new-build coordinator's recovery would count as a clean ack. Finish upgrading both central nodes, then deploy. - **Mixed-version fleets.** An old node never sets the flag, so its acks are treated exactly as before (clean). A new node acking an old central: the old coordinator ignores the unknown field and seals as it always did — no worse than today. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/FailureReasons.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/FailureReasons.cs new file mode 100644 index 000000000..5a1dac84e --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/FailureReasons.cs @@ -0,0 +1,20 @@ +namespace ZB.MOM.WW.OtOpcUa.Configuration; + +/// +/// The FailureReason column bound shared by Deployment and NodeDeploymentState +/// (nvarchar(2048), ). Every writer of either column goes through +/// : a reason built from free text (an exception message, a node's reported reason) that +/// overflows it makes SQL Server refuse the whole SaveChanges, and a write that can never succeed wedges +/// whatever is retrying it (lmxopcua#868). +/// +public static class FailureReasons +{ + /// The column's maximum length. + public const int MaxLength = 2048; + + /// Bounds a reason to (ellipsis-terminated when cut); null stays null. + /// The reason, or null. + /// The reason, fitting the column. + public static string? Truncate(string? reason) => + reason is null || reason.Length <= MaxLength ? reason : reason[..(MaxLength - 1)] + "…"; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 3e0cb2c17..898c6e282 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -109,6 +109,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private readonly Dictionary _unpersisted = new(NodeIdComparer); private string? _lastPersistError; + /// When the current deadline was first deferred on an unpersisted ack (bounds the deferral). + private DateTime? _deadlineDeferredSinceUtc; + + /// How long a deadline may wait on unpersisted acks before deciding from memory (10 × the retry ceiling). + private TimeSpan MaxDeadlineDeferral => TimeSpan.FromTicks(_loadRetryMax.Ticks * 10); + /// /// Where central→node commands are handed off (per-cluster mesh Phase 2), or /// to publish on the mediator directly. @@ -419,11 +425,15 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU _log.Warning( "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + - "re-dispatching with {Remaining} of the apply deadline left", + "re-dispatching with a fresh apply deadline ({Remaining} of the original was left)", _current, remaining); - // The REMAINING time (review L4): CreatedAtUtc never moves, so a re-drive must not restart the clock. + // A FULL deadline (third review #3, superseding the second review's "remaining time"): a node needs its + // whole outcome timeout (DeployTimings.ApplyDeadline - ApplyOutcomeSafetyMargin) plus that margin to + // report — i.e. the full ApplyDeadline — so ANY remaining time shorter than that let the nodes apply + // and the deployment time out moments later. Bounded anyway: there is no dispatched-at column, so a + // SECOND restart recomputes from CreatedAtUtc and, past the deadline by then, closes the row instead. HandleDispatch(new DispatchDeployment( - _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()), remaining); + _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); } else { @@ -468,11 +478,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU _current, _acks.Count, _expectedAcks.Count); } - private void HandleDispatch(DispatchDeployment msg) => HandleDispatch(msg, deadline: null); - - /// The dispatch. - /// The apply deadline to arm; null = the full . - private void HandleDispatch(DispatchDeployment msg, TimeSpan? deadline) + private void HandleDispatch(DispatchDeployment msg) { // A duplicate of the deployment already in flight with a seeded expected set (a dispatch queued while the // recovery load ran, for the very row that load re-drove) must not reset the acks already gathered. @@ -567,7 +573,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU // directly, so this actor no longer knows which transport is in force. if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg)); else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg)); - Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), deadline ?? _applyDeadline); + Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), _applyDeadline); if (_expectedAcks.Count == 0) { @@ -621,6 +627,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU { PersistNodeAck(msg); } + catch (Exception ex) when (IsPoison(ex)) + { + // Third review #1: a write the DB REJECTS (constraint, truncation, …) fails identically on every retry; + // retrying it forever wedged the deployment (the deadline deferred on it) and every later deploy. The + // node DID ack, so count it from memory and say loudly that its row is not recorded. + _log.Error(ex, + "The ApplyAck from {Node} for {Id} ({Outcome}) was REJECTED by the ConfigDb and will not be retried; " + + "counting it from memory. Its NodeDeploymentState row stays unrecorded", + msg.NodeId, msg.DeploymentId, msg.Outcome); + DropUnpersisted(msg.NodeId); + CountAck(msg); + return true; + } catch (Exception ex) { var attempts = _unpersisted.TryGetValue(msg.NodeId, out var prior) ? prior.Attempts + 1 : 1; @@ -639,17 +658,42 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU return false; } - if (_unpersisted.Remove(msg.NodeId)) - { - Timers.Cancel(AckRetryTimerKey(msg.NodeId)); - if (_unpersisted.Count == 0) _lastPersistError = null; - ReportPersistBacklog(); - } + DropUnpersisted(msg.NodeId); + CountAck(msg); + return true; + } + private void CountAck(ApplyAck msg) + { _acks[msg.NodeId] = msg.Outcome; // An old node never sets Degraded, so its ack takes the clean branch exactly as before. if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId); else _degraded.Remove(msg.NodeId); + } + + private void DropUnpersisted(NodeId node) + { + if (!_unpersisted.Remove(node)) return; + Timers.Cancel(AckRetryTimerKey(node)); + if (_unpersisted.Count == 0) _lastPersistError = null; + ReportPersistBacklog(); + } + + /// + /// A non-transient rejection of the write itself: an EF (not a concurrency + /// conflict) with no transient database error, timeout or socket failure underneath. Connectivity failures + /// surface as other exception types (or as transient s) and stay + /// retryable. + /// + internal static bool IsPoison(Exception ex) + { + if (ex is not DbUpdateException || ex is DbUpdateConcurrencyException) return false; + for (var e = ex.InnerException; e is not null; e = e.InnerException) + { + if (e is System.Data.Common.DbException { IsTransient: true } + or TimeoutException or System.Net.Sockets.SocketException or System.IO.IOException) + return false; + } return true; } @@ -697,18 +741,17 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU ? Truncate(DegradedReasonPrefix + (string.IsNullOrWhiteSpace(msg.DegradedReason) ? "address-space apply degraded (no reason given)" : msg.DegradedReason)) - : msg.FailureReason; + // Third review #1: a Failed ack's reason is the node's free text (often an exception message) and was + // written UNBOUNDED — on SQL Server an overflow refuses the SaveChanges forever. + : FailureReasons.Truncate(msg.FailureReason); db.SaveChanges(); } private static bool IsDegradedAck(ApplyAck msg) => msg.Outcome == ApplyAckOutcome.Applied && msg.Degraded; // FailureReason is nvarchar(2048) on both tables (OtOpcUaConfigDbContext); a reason built from a node's - // free text must never fail the SaveChanges that records the ack. - private const int FailureReasonMaxLength = 2048; - - private static string Truncate(string s) => - s.Length <= FailureReasonMaxLength ? s : s[..(FailureReasonMaxLength - 1)] + "…"; + // free text must never fail the SaveChanges that records the ack. One shared bound (FailureReasons). + private static string Truncate(string s) => FailureReasons.Truncate(s)!; private void SealDeployment() { @@ -763,6 +806,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU if (_unpersisted.Count > 0) { foreach (var pending in _unpersisted.Values.ToList()) TryPersistAck(pending.Ack); + _deadlineDeferredSinceUtc ??= DateTime.UtcNow; + if (_unpersisted.Count > 0 && DateTime.UtcNow - _deadlineDeferredSinceUtc.Value >= MaxDeadlineDeferral) + { + // Third review #1: bounded. Past the cap the outcome is decided from what was RECEIVED — the nodes did + // ack — rather than waiting on rows that may never be writable. + _log.Error( + "Deployment {Id}: {Count} received ack(s) still could not be persisted after deferring the deadline " + + "{Cap}; deciding the outcome from the acks held in memory (last error: {Error})", + _current.Value, _unpersisted.Count, MaxDeadlineDeferral, _lastPersistError); + foreach (var pending in _unpersisted.Values.ToList()) + { + DropUnpersisted(pending.Ack.NodeId); + CountAck(pending.Ack); + } + } if (_unpersisted.Count > 0) { var delay = Backoff(_unpersisted.Values.Min(p => p.Attempts)); @@ -841,6 +899,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU private void ResetForNext() { Timers.Cancel(DeadlineTimerKey); + _deadlineDeferredSinceUtc = null; foreach (var node in _unpersisted.Keys) Timers.Cancel(AckRetryTimerKey(node)); if (_unpersisted.Count > 0) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 56ea2ad3b..b16f68634 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -295,7 +295,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// idempotent "already at this revision" short-circuit: the short-circuit would ack a clean Applied /// for an address space known to be partial, and the re-apply is what repairs it (a degraded /// non-rebuild apply leaves the publish actor's composition un-advanced, so it re-diffs the missing - /// nodes). Reset by every clean outcome and by every boot path that restores a revision. + /// nodes). Reset by every clean outcome and by the ConfigDb boot path (an Applied row is a final outcome); + /// SET by the cache boot paths, whose cached revision may never have reported an outcome (lmxopcua#868). /// private bool _currentRevisionDegraded; @@ -1434,6 +1435,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS } _currentRevision = revision; + // Unconfirmed (lmxopcua#868 third review #2): the cache is written BEFORE the address-space outcome, so a + // restart in that window boots a revision central never got an ack for. Marking it unconfirmed makes a + // re-dispatch of it RE-APPLY (and ack the real outcome) instead of short-circuiting a clean Applied. + _currentRevisionDegraded = true; BecomeSteady(); _log.Info( @@ -1514,6 +1519,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // Stale would have started does not run here, so recovery rides on the next dispatch — // matching how a normally-booted node behaves. _currentRevision = revision; + // Unconfirmed, as in BootstrapFromCache (third review #2): a re-dispatch of it re-applies. + _currentRevisionDegraded = true; BecomeSteady(); _isRunningFromCache = true; @@ -4336,6 +4343,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS private void UpsertNodeDeploymentState(DeploymentId deploymentId, NodeDeploymentStatus status, string? failureReason) { + // nvarchar(2048): an exception message or a degraded reason can exceed it, and SQL Server then refuses the + // write — the row would stay Applying forever (lmxopcua#868 third review #1). + failureReason = FailureReasons.Truncate(failureReason); // Per-cluster mesh Phase 4: a driver-only node has NO ConfigDb. Central is the ack system of // record — ConfigPublishCoordinator.PersistNodeAck already writes NodeDeploymentState from every // ApplyAck it receives (and seeds the Applying rows at dispatch) — so this node's own upsert is diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs index bb3045a67..202969bc7 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs @@ -402,6 +402,115 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); } + /// + /// Third review #1: a Failed ack's reason (node free text) was written unbounded; on SQL Server the overflow + /// refuses the SaveChanges forever and the deployment wedged. It is truncated to the column. + /// + [Fact] + public void An_oversized_failure_reason_is_truncated_not_retried_forever() + { + var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout); + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed, + new string('x', 5000), CorrelationId.NewId())); + + AwaitAssert(() => + { + using var ctx = db.CreateDbContext(); + ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.PartiallyFailed); + var row = ctx.NodeDeploymentStates.Single(); + row.Status.ShouldBe(NodeDeploymentStatus.Failed); + row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Third review #1: a write the DB REJECTS (non-transient DbUpdateException) is not retried forever — the ack is + /// counted from memory and the deployment completes. + /// + [Fact] + public void A_rejected_ack_write_is_counted_from_memory_not_retried_forever() + { + var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout); + db.PoisonNodeStateUpdates = true; + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var ctx = db.CreateDbContext(); + ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Third review #1: a write that keeps failing transiently defers the deadline only up to a cap + /// (10 × the retry ceiling); past it the outcome is decided from the acks held in memory. + /// + [Fact] + public void Deadline_deferral_on_unpersisted_acks_is_bounded() + { + var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(TimeSpan.FromSeconds(1)); + db.FailNodeStateUpdatesTransiently = true; + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + // Deadline 1 s + cap 10 × 200 ms: decided within a few seconds, never left in flight. + AwaitAssert(() => + { + using var ctx = db.CreateDbContext(); + ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// + /// Third review #3: a lost dispatch recovered with only a sliver of its deadline left used to be re-driven with + /// that sliver — the nodes applied and the deployment timed out moments later. It gets a full deadline. + /// + [Fact] + public void A_lost_dispatch_redriven_near_its_deadline_gets_a_full_deadline() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, Node); + var deadline = TimeSpan.FromSeconds(3); + var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow - TimeSpan.FromSeconds(2.5)); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline)); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + + // Well past the 0.5 s that was left, still inside a fresh 3 s deadline. + Thread.Sleep(TimeSpan.FromSeconds(1.5)); + using (var db = dbFactory.CreateDbContext()) + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + + coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + private (ConstraintEnforcingDbFactory Db, DeploymentId Id, IActorRef Coordinator) DispatchedOnConstraintDb(TimeSpan applyDeadline) + { + var db = new ConstraintEnforcingDbFactory(Guid.NewGuid().ToString("N")); + SeedNodes(db, Node); + var deploymentId = SeedDeployment(db, DeploymentStatus.Dispatching, DateTime.UtcNow); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props( + db, applyDeadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax)); + coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var ctx = db.CreateDbContext(); + ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + return (db, deploymentId, coordinator); + } + /// A single-node deployment dispatched and AwaitingApplyAcks through an outage-capable factory. private (IDbContextFactory Direct, OutageDbFactory Outage, DeploymentId Id, IActorRef Coordinator, ConfigPublishCoordinatorReadiness Readiness) DispatchedSingleNode(TimeSpan applyDeadline) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs new file mode 100644 index 000000000..a3d29603b --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs @@ -0,0 +1,69 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +/// +/// An in-memory ConfigDb that behaves like SQL Server where the InMemory provider does not (lmxopcua#868 third +/// review): a FailureReason longer than the column refuses the whole SaveChanges with a +/// . It can also reject (, a non-transient +/// ) or fail transiently () every +/// update of an existing NodeDeploymentState row. +/// +public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFactory +{ + private readonly Interceptor _interceptor = new(); + + /// Every update of an existing node row is rejected as a constraint violation. + public bool PoisonNodeStateUpdates { get => _interceptor.Poison; set => _interceptor.Poison = value; } + + /// Every update of an existing node row fails as if the connection dropped. + public bool FailNodeStateUpdatesTransiently { get => _interceptor.Transient; set => _interceptor.Transient = value; } + + /// + public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName) + .AddInterceptors(_interceptor) + .Options); + + private sealed class Interceptor : SaveChangesInterceptor + { + public volatile bool Poison; + public volatile bool Transient; + + public override InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + Check(eventData.Context!); + return result; + } + + public override ValueTask> SavingChangesAsync( + DbContextEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default) + { + Check(eventData.Context!); + return ValueTask.FromResult(result); + } + + private void Check(DbContext ctx) + { + foreach (var e in ctx.ChangeTracker.Entries()) + { + var reason = e.Entity switch + { + NodeDeploymentState n => n.FailureReason, + Deployment d => d.FailureReason, + _ => null, + }; + if (reason is { Length: > FailureReasons.MaxLength }) + throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'."); + if (e.Entity is NodeDeploymentState && e.State == EntityState.Modified) + { + if (Poison) throw new DbUpdateException("The UPDATE statement conflicted with a CHECK constraint."); + if (Transient) throw new InvalidOperationException("A network-related error occurred (simulated)."); + } + } + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs index 23c2d4195..033c22934 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs @@ -56,6 +56,31 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase cache.UnkeyedReads.ShouldBe(1); } + /// + /// lmxopcua#868 third review #2: the cache is written before the address-space outcome, so a cache-booted + /// revision may never have been acked. A re-dispatch of it must RE-APPLY (here: fail honestly, since the + /// ConfigDb is unreachable) — never short-circuit a clean Applied for an outcome nobody confirmed. + /// + [Fact] + public void A_cache_booted_revision_is_unconfirmed_so_a_redispatch_reapplies_instead_of_acking_clean() + { + var cached = new CachedDeploymentArtifact( + DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow.AddMinutes(-1)); + var coordinator = CreateTestProbe(); + var actor = Sys.ActorOf(DriverHostActor.Props( + new ThrowingDbFactory(), TestNode, ackRouter: coordinator.Ref, + localRoles: new HashSet { "driver" }, + deploymentArtifactCache: new StubArtifactCache(cached))); + AskDiagnostics(actor).CurrentRevision.ShouldBe(CachedRev); + + actor.Tell(new ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.DispatchDeployment( + DeploymentId.NewId(), CachedRev, CorrelationId.NewId())); + + var ack = coordinator.ExpectMsg( + PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + ack.Outcome.ShouldNotBe(ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.ApplyAckOutcome.Applied); + } + [Fact] public void BootFromCache_HandsTheCachedArtifactToTheAddressSpaceRebuild() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs index 99f261ee1..0875d1755 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs @@ -238,6 +238,58 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase }, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); } + [Fact] + public void An_oversized_degraded_reason_is_truncated_so_the_local_row_still_records_the_outcome() + { + // lmxopcua#868 third review #1: the row's FailureReason is nvarchar(2048). An unbounded reason made SQL Server + // refuse the write (logged and swallowed), so the row stayed Applying forever. A DB that enforces the column. + var db = new LengthEnforcingDbFactory(Guid.NewGuid().ToString("N")); + var deploymentId = V3ArtifactSeeder.SeedV3Deployment(db, RevA, + rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) }, + unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) }); + var coordinator = CreateTestProbe(); + var publish = CreateTestProbe(); + var actor = Sys.ActorOf(DriverHostActor.Props( + db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, + opcUaPublishActor: publish.Ref)); + + actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); + var rebuild = publish.FishForMessage(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, new string('f', 5000))); + coordinator.ExpectMsg(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeTrue(); + + AwaitAssert(() => + { + using var ctx = db.CreateDbContext(); + var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value); + row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied); + row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength); + row.FailureReason.ShouldStartWith(ApplyAck.DegradedReasonPrefix); + }, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); + } + + /// In-memory ConfigDb that refuses an over-length FailureReason, as SQL Server does. + private sealed class LengthEnforcingDbFactory(string dbName) : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName) + .AddInterceptors(new Enforcer()) + .Options); + + private sealed class Enforcer : Microsoft.EntityFrameworkCore.Diagnostics.SaveChangesInterceptor + { + public override Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult SavingChanges( + Microsoft.EntityFrameworkCore.Diagnostics.DbContextEventData eventData, + Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult result) + { + foreach (var e in eventData.Context!.ChangeTracker.Entries()) + if (e.Entity.FailureReason is { Length: > FailureReasons.MaxLength }) + throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'."); + return result; + } + } + } + [Fact] public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin() { From 13cc7b8353d8e2ea8ffb7e9bcc50fae81726b691 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:16:01 -0500 Subject: [PATCH 26/28] fix(galaxy): #865 review round 6 - ignore stream faults from a superseded pump Each EventPump's fault callback now carries the pump's identity; a fault from a pump that is no longer the live one (or belongs to an older registry session) is logged at Debug and not escalated, so a stale pump's late fault cannot fail a good replay now that a failure during Replaying fails the attempt. The replay-retry warning includes the fault. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 24 ++++++++++++++++--- .../Runtime/ReconnectSupervisor.cs | 5 +++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 2cfc682e4..e548a138a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -2162,12 +2162,16 @@ public sealed class GalaxyDriver "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync."); } if (_eventPump is not null) return _eventPump; - _eventPump = new EventPump( + // The fault callback carries the pump's identity, so a late fault from a pump that is no + // longer the live one (a stale pump of a reopened session) is not escalated (#865 review). + EventPump? created = null; + created = new EventPump( _subscriber!, _subscriptions, _logger, channelCapacity: _options.MxAccess.EventPumpChannelCapacity, clientName: _options.MxAccess.ClientName, - onStreamFault: OnEventPumpStreamFault, + onStreamFault: cause => OnEventPumpStreamFault(created, cause), onReplayGap: OnEventPumpReplayGap); + _eventPump = created; _eventPump.OnDataChange += OnPumpDataChange; _eventPump.Start(); return _eventPump; @@ -2441,7 +2445,7 @@ public sealed class GalaxyDriver /// replay; without this hand-off a transient transport drop permanently kills /// the event stream and GetHealth() keeps reporting Healthy. /// - private void OnEventPumpStreamFault(Exception cause) + private void OnEventPumpStreamFault(EventPump? source, Exception cause) { if (_runtimeStopping) { @@ -2451,6 +2455,20 @@ public sealed class GalaxyDriver return; } + if (source is not null) + { + bool stale; + lock (_pumpLock) stale = !ReferenceEquals(_eventPump, source) || source.Session != _subscriptions.CurrentSession; + if (stale) + { + // A pump that is no longer the live one (its session was reopened; the replay replaces or + // already replaced it). Its fault says nothing about the current transport — and reported + // while Replaying it would fail a good replay (#865 review). + _logger.LogDebug(cause, "GalaxyDriver {InstanceId} ignored a stream fault from a superseded event pump.", _driverInstanceId); + return; + } + } + var supervisor = _supervisor; if (supervisor is null) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs index 43ab2d920..d6ca1318f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs @@ -271,9 +271,11 @@ public sealed class ReconnectSupervisor : IDisposable // === Done — unless the transport failed again while the replay ran === bool faultedDuringReplay; + string? replayFault; lock (_stateLock) { faultedDuringReplay = _faultDuringReplay; + replayFault = _lastError; if (!faultedDuringReplay) { _lastError = null; @@ -285,7 +287,8 @@ public sealed class ReconnectSupervisor : IDisposable // Going Healthy here would hide a dead transport (the fault was swallowed above), and the // next report would restart at attempt 1 with no backoff. Retry this cycle with backoff. _logger.LogWarning( - "Galaxy replay completed but the transport failed again while it ran (attempt {Attempt}) — retrying", attempt); + "Galaxy replay completed but the transport failed again while it ran (attempt {Attempt}: {Fault}) — retrying", + attempt, replayFault); continue; } _logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt); From 59d33ab93805617305ebb5fcc838b64d350cf0c8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:20:49 -0500 Subject: [PATCH 27/28] test(galaxy): #865 review round 7 - pin the superseded-pump fault filter Adds an escalated-stream-fault test seam and a test: the live pump's fault escalates (positive controls before and after), a superseded pump's late fault after a reopen does not; the absence is asserted only once both pumps' callbacks are known to have run (HasEnded). Verified to fail with the filter disabled. The filter's comment now says the identity check does the work and warns about a future session bump while Healthy. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../GalaxyDriver.cs | 14 ++++-- .../Runtime/ProbeInitialValueTests.cs | 45 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index e548a138a..fb301e855 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -2445,6 +2445,11 @@ public sealed class GalaxyDriver /// replay; without this hand-off a transient transport drop permanently kills /// the event stream and GetHealth() keeps reporting Healthy. /// + private int _escalatedStreamFaults; + + /// Test seam (#865): stream faults that passed the superseded-pump filter (escalated toward the supervisor). + internal int EscalatedStreamFaultsForTests => Volatile.Read(ref _escalatedStreamFaults); + private void OnEventPumpStreamFault(EventPump? source, Exception cause) { if (_runtimeStopping) @@ -2461,13 +2466,16 @@ public sealed class GalaxyDriver lock (_pumpLock) stale = !ReferenceEquals(_eventPump, source) || source.Session != _subscriptions.CurrentSession; if (stale) { - // A pump that is no longer the live one (its session was reopened; the replay replaces or - // already replaced it). Its fault says nothing about the current transport — and reported - // while Replaying it would fail a good replay (#865 review). + // A pump that is no longer the live one: the identity check does the work (the replay + // replaces/replaced it). The session part only matters between a reopen and its replay, + // where the supervisor ignores faults anyway — ⚠️ a future session bump while Healthy that + // keeps the same pump would make its real faults look stale here. Its fault says nothing + // about the current transport, and reported while Replaying it would fail a good replay. _logger.LogDebug(cause, "GalaxyDriver {InstanceId} ignored a stream fault from a superseded event pump.", _driverInstanceId); return; } } + Interlocked.Increment(ref _escalatedStreamFaults); var supervisor = _supervisor; if (supervisor is null) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs index 7eabd47ab..e496d7fae 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ProbeInitialValueTests.cs @@ -116,6 +116,12 @@ public sealed class ProbeInitialValueTests } } + /// Fault the currently attached stream (the gateway dropping it). + public void FaultAttachedStream(Exception cause) + { + lock (_gate) _attached.ShouldNotBeNull("a stream must be attached").Writer.TryComplete(cause); + } + public string[] CallsSnapshot() { lock (_gate) return [.. Calls]; @@ -349,6 +355,45 @@ public sealed class ProbeInitialValueTests "a replay must not report success over a dead event stream"); } + /// + /// #865 review: a stream fault is escalated only from the LIVE pump. A fault from a pump superseded + /// by a reopen (its session is no longer current) is ignored — it would fail a good replay — while + /// the live pump's faults (before the reopen, and the replay's new pump after it) still escalate. + /// + [Fact] + public async Task Only_the_live_pumps_stream_faults_are_escalated() + { + var sub = new GatewayLikeSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) + { + RecreateOverrideForTests = _ => Task.CompletedTask, + }; + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // pump 1 + + // Positive control: the live pump's fault escalates. + sub.FaultAttachedStream(new IOException("drop 1")); + await TestWait.UntilAsync(() => driver.EscalatedStreamFaultsForTests == 1, "the live pump's fault must escalate"); + + // Recover onto pump 2 (the replay restarts the faulted pump), then reopen: pump 2 is now superseded. + await driver.InvokeReplayForTestAsync(CancellationToken.None); + var pump2 = driver.EventPumpInstanceForTests.ShouldNotBeNull(); + await driver.InvokeReopenForTestAsync(CancellationToken.None); + sub.FaultAttachedStream(new IOException("late fault from the superseded pump")); + await TestWait.UntilAsync(() => ((EventPump)pump2).HasEnded, "the superseded pump's stream must have faulted"); + + // Positive control after the absence: the replay's new pump faults and that DOES escalate — so the + // count can only be 2 if the superseded pump's fault was ignored. + await driver.InvokeReplayForTestAsync(CancellationToken.None); + var pump3 = (EventPump)driver.EventPumpInstanceForTests.ShouldNotBeNull(); + pump3.ShouldNotBeSameAs(pump2); + sub.FaultAttachedStream(new IOException("drop 3")); + // HasEnded is set only after the fault callback ran, so once both pumps have ended every escalation + // that will ever happen has happened. + await TestWait.UntilAsync(() => pump3.HasEnded, "the new live pump's stream must have faulted"); + driver.EscalatedStreamFaultsForTests.ShouldBe(2, "the new live pump's fault escalates; the superseded pump's did not"); + } + /// /// A probe sync on a driver whose pump was never started (no platform source, e.g. the /// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just From a8f4be616ba6749d953b999f6574c60d40929d6c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 13:20:58 -0500 Subject: [PATCH 28/28] test(opcua): opt-in perf harness for #855 folder event per-item delivery FolderEventDeliveryPerfTests (Category=Perf, skipped unless OTOPCUA_EVENT_PERF=1) reports per-alarm-report node-manager Lock hold and end-to-end delivery latency for one equipment folder with N = 1/10/50/200 event items, ACL on/off, folder subscribers publishing or stalled at a full queue; the latency of an i=2253 subscriber while folder subscribers stall; and a QueueSize sensitivity sweep. Compiles against the pre-#855 tree for a before/after comparison. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../FolderEventDeliveryPerfTests.cs | 350 ++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs new file mode 100644 index 000000000..25b14788e --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs @@ -0,0 +1,350 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text; +using Opc.Ua; +using Opc.Ua.Client; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using MonitoredItem = Opc.Ua.Client.MonitoredItem; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; + +/// +/// Gitea #855 cost measurement: since #855 a condition event is queued to every event item on an equipment folder by +/// the reporter itself, under the node manager's Lock (per-item verdict, QueueEvent with the SDK's +/// filter evaluation, duplicate scan over the last 1000 queued events and a List.RemoveAt(0) at a full queue). +/// Before #855 that work ran on the SDK's per-node channel consumer, off Lock. This harness reports, per alarm +/// report, the Lock hold time and the end-to-end delivery latency, for one equipment folder carrying N event +/// items, ACL on / off, with the folder's subscribers publishing or stalled (publishing disabled, item queues full and +/// discarding oldest) — and the latency of a Server-object (i=2253) subscriber while the folder's subscribers +/// are stalled (the ScadaBridge shape). +/// +/// +/// Measurement only: nothing is asserted beyond the harness's own sanity checks. Opt-in and never part of a +/// default run: every case skips unless OTOPCUA_EVENT_PERF=1. +/// Run: OTOPCUA_EVENT_PERF=1 dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests -c Release +/// --filter "FullyQualifiedName~FolderEventDeliveryPerfTests" --logger "console;verbosity=detailed". +/// OTOPCUA_EVENT_PERF_REPORTS sets the measured reports per case (default 1000) and +/// OTOPCUA_EVENT_PERF_OUT appends each result row to a file. +/// The rig is the same one uses and existed before #855, so this file +/// compiles against the pre-#855 tree for a before/after comparison. +/// +[Trait("Category", "Perf")] +public sealed class FolderEventDeliveryPerfTests +{ + private const string OptInVariable = "OTOPCUA_EVENT_PERF"; + private const string EqA = ServerEventRig.EqA; + private const uint DefaultQueueSize = 1000; + private const double PublishingIntervalMs = 50; + + private static int Reports => + int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_EVENT_PERF_REPORTS"), out var r) && r > 0 ? r : 1000; + + private static void RequireOptIn() + { + if (Environment.GetEnvironmentVariable(OptInVariable) != "1") + Assert.Skip($"perf harness: set {OptInVariable}=1 to run (not part of a default run)"); + } + + /// + /// Lock hold per report and (publishing only) the time until the LAST of the N folder items received the event. + /// + [Theory] + [InlineData(1, false, false)] + [InlineData(10, false, false)] + [InlineData(50, false, false)] + [InlineData(200, false, false)] + [InlineData(1, true, false)] + [InlineData(10, true, false)] + [InlineData(50, true, false)] + [InlineData(200, true, false)] + [InlineData(1, false, true)] + [InlineData(10, false, true)] + [InlineData(50, false, true)] + [InlineData(200, false, true)] + [InlineData(1, true, true)] + [InlineData(10, true, true)] + [InlineData(50, true, true)] + [InlineData(200, true, true)] + public Task Folder_items(int n, bool acl, bool stalled) => RunAsync(n, acl, stalled, serverObjectProbe: false); + + /// + /// A publishing Server-object (i=2253) subscriber on its own session, while N folder items on another + /// session are stalled with full queues. N = 0 is the baseline. + /// + [Theory] + [InlineData(0, false)] + [InlineData(50, false)] + [InlineData(200, false)] + [InlineData(0, true)] + [InlineData(50, true)] + [InlineData(200, true)] + public Task Server_object_subscriber_while_folder_items_stall(int n, bool acl) => + RunAsync(n, acl, stalled: true, serverObjectProbe: true); + + /// + /// Full-queue sensitivity: N = 50 stalled items at different QueueSizes. At a full queue every + /// QueueEvent scans the last min(QueueSize, 1000) queued events for a duplicate and shifts the whole queue + /// (List.RemoveAt(0)), so the per-item cost grows with the queue, not with N. + /// + [Theory] + [InlineData(10u)] + [InlineData(100u)] + [InlineData(1000u)] + [InlineData(10000u)] + public Task Stalled_items_queue_size_sensitivity(uint queueSize) => + RunAsync(50, acl: false, stalled: true, serverObjectProbe: false, queueSize); + + private static async Task RunAsync( + int n, bool acl, bool stalled, bool serverObjectProbe, uint queueSize = DefaultQueueSize) + { + RequireOptIn(); + var ct = TestContext.Current.CancellationToken; + await using var rig = await ServerEventRig.StartAsync($"Perf855-{n}", encrypted: false, ct); + rig.KeyRoles = []; + rig.MaterialiseNative(); + rig.Enforce(acl, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); + + var reports = Reports; + var reportedAt = new long[reports]; + var receipts = new Receipts(reports); + + ISession? folderSession = null; + ISession? serverSession = null; + try + { + Subscription? folderSubscription = null; + var folderHandles = new HashSet(); + if (n > 0) + { + folderSession = await rig.ConnectAsync(anonymous: false, ct); + folderSubscription = await SubscribeAsync(folderSession, receipts, isServer: false, ct); + for (var i = 0; i < n; i++) + { + var item = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA)); + item.QueueSize = queueSize; + folderHandles.Add(item.ClientHandle); + } + + await folderSubscription.ApplyChangesAsync(ct); + folderSubscription.MonitoredItems.ShouldAllBe(m => m.Status.Error == null); + // The server's revision (MaxEventQueueSize) — what the measured queues actually hold. + queueSize = folderSubscription.MonitoredItems.First().Status.QueueSize; + } + + receipts.FolderHandles = folderHandles; + + MonitoredItem? onServer = null; + if (serverObjectProbe) + { + serverSession = await rig.ConnectAsync(anonymous: false, ct); + var serverSubscription = await SubscribeAsync(serverSession, receipts, isServer: true, ct); + onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server); + await serverSubscription.ApplyChangesAsync(ct); + onServer.Status.Error.ShouldBeNull(); + receipts.ServerHandle = onServer.ClientHandle; + } + + if (stalled && folderSubscription is not null) + { + await folderSubscription.SetPublishingModeAsync(false, ct); + + // Fill every folder item's queue to QueueSize (and past it, so the measured reports run at a full queue, + // discarding oldest), then let any off-Lock consumer (pre-#855: the SDK's channel task) catch up. + for (var i = 0; i < queueSize + 50; i++) rig.Project(native: true, State($"w{i}")); + await SettleAsync(rig, n, ct); + } + + // Warm-up (JIT, first-verdict caches) — not measured. + for (var i = 0; i < 50; i++) rig.Project(native: true, State($"x{i}")); + await SettleAsync(rig, n, ct); + + var holds = new double[reports]; + for (var i = 0; i < reports; i++) + { + var state = State($"p{i}"); + long t0, t1; + lock (rig.Nm.Lock) + { + t0 = Stopwatch.GetTimestamp(); + reportedAt[i] = t0; + rig.Project(native: true, state); + t1 = Stopwatch.GetTimestamp(); + } + + holds[i] = Stopwatch.GetElapsedTime(t0, t1).TotalMicroseconds; + // ~1 kHz report rate: an alarm storm, not a single burst, so a consumer that keeps up is not penalised. + Thread.Sleep(1); + } + + // Wait for the tail of every probe that is expected to receive. + var expectFolder = n > 0 && !stalled; + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline + && ((expectFolder && !receipts.FolderComplete(reports - 1, n)) + || (serverObjectProbe && !receipts.ServerHas(reports - 1)))) + { + await Task.Delay(50, ct); + } + + var deferredPeak = rig.Nm.DeferredEventReports; + var row = new StringBuilder(); + row.Append(CultureInfo.InvariantCulture, + $"{(serverObjectProbe ? "i=2253" : "folder"),-6} N={n,-3} acl={(acl ? "on " : "off")} {(stalled ? "stalled " : "publishing")} q={queueSize,-5} | "); + row.Append(CultureInfo.InvariantCulture, $"lock hold us p50 {Pct(holds, 50),8:F1} p90 {Pct(holds, 90),8:F1} p99 {Pct(holds, 99),8:F1} max {holds.Max(),9:F1} mean {holds.Average(),8:F1}"); + row.Append(CultureInfo.InvariantCulture, $" | per-item mean {(n > 0 ? holds.Average() / n : 0),6:F2} us"); + if (expectFolder) + { + var lat = receipts.FolderLatenciesMs(reportedAt, n); + row.Append(CultureInfo.InvariantCulture, + $" | folder all-N latency ms p50 {Pct(lat, 50),6:F1} p99 {Pct(lat, 99),6:F1} max {Max(lat),7:F1} (got {lat.Length}/{reports})"); + } + + if (serverObjectProbe) + { + var lat = receipts.ServerLatenciesMs(reportedAt); + row.Append(CultureInfo.InvariantCulture, + $" | i=2253 latency ms p50 {Pct(lat, 50),6:F1} p99 {Pct(lat, 99),6:F1} max {Max(lat),7:F1} (got {lat.Length}/{reports})"); + } + + row.Append(CultureInfo.InvariantCulture, $" | deferred-now {deferredPeak}"); + Report(row.ToString()); + } + finally + { + folderSession?.Dispose(); + serverSession?.Dispose(); + } + } + + /// + /// Gives an off-Lock consumer (pre-#855) time to drain what was just reported; post-#855 nothing is pending. + /// + private static async Task SettleAsync(ServerEventRig rig, int n, CancellationToken ct) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60); + while (rig.Nm.DeferredEventReports > 0 && DateTime.UtcNow < deadline) await Task.Delay(50, ct); + await Task.Delay(TimeSpan.FromMilliseconds(500 + (10 * n)), ct); + } + + private static async Task SubscribeAsync(ISession session, Receipts receipts, bool isServer, CancellationToken ct) + { + var subscription = new Subscription(session.DefaultSubscription) + { + PublishingInterval = (int)PublishingIntervalMs, + MaxNotificationsPerPublish = 0, + }; + subscription.FastEventCallback = receipts.OnEvents; + session.AddSubscription(subscription); + await subscription.CreateAsync(ct); + return subscription; + } + + private static double Pct(double[] values, double p) + { + if (values.Length == 0) return double.NaN; + var sorted = values.Order().ToArray(); + var idx = (int)Math.Ceiling(p / 100.0 * sorted.Length) - 1; + return sorted[Math.Clamp(idx, 0, sorted.Length - 1)]; + } + + private static double Max(double[] values) => values.Length == 0 ? double.NaN : values.Max(); + + private static void Report(string text) + { + TestContext.Current.TestOutputHelper?.WriteLine(text); + if (Environment.GetEnvironmentVariable("OTOPCUA_EVENT_PERF_OUT") is { Length: > 0 } path) + File.AppendAllText(path, text + Environment.NewLine); + } + + private static AlarmConditionSnapshot State(string message) => + new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message); + + private static NodeAcl KeyEquipmentGrant(string equipment) => new() + { + NodeAclId = $"acl-key-{equipment}", + ClusterId = ServerEventRig.Cluster, + Principal = ServerEventRig.KeyPrincipal, + ScopeKind = NodeAclScopeKind.Equipment, + ScopeId = $"eq-{equipment}", + PermissionFlags = NodePermissions.ReadOnly, + }; + + /// Client-side receipt times of the measured (p{i}) events, per probe. + private sealed class Receipts(int reports) + { + private readonly object _gate = new(); + private readonly int[] _folderCount = new int[reports]; + private readonly long[] _folderLast = new long[reports]; + private readonly long[] _serverFirst = new long[reports]; + + public IReadOnlySet FolderHandles { get; set; } = new HashSet(); + + public uint? ServerHandle { get; set; } + + public void OnEvents(Subscription subscription, EventNotificationList notification, ArrayOf stringTable) + { + var now = Stopwatch.GetTimestamp(); + lock (_gate) + { + foreach (var e in notification.Events) + { + var message = RecordedEvents.Message(e.EventFields); + if (message is not { Length: > 1 } || message[0] != 'p') continue; + if (!int.TryParse(message.AsSpan(1), NumberStyles.Integer, CultureInfo.InvariantCulture, out var i) + || i < 0 || i >= _folderCount.Length) + { + continue; + } + + if (ServerHandle == e.ClientHandle) + { + if (_serverFirst[i] == 0) _serverFirst[i] = now; + } + else if (FolderHandles.Contains(e.ClientHandle)) + { + _folderCount[i]++; + _folderLast[i] = now; + } + } + } + } + + public bool FolderComplete(int i, int n) + { + lock (_gate) return _folderCount[i] >= n; + } + + public bool ServerHas(int i) + { + lock (_gate) return _serverFirst[i] != 0; + } + + public double[] FolderLatenciesMs(long[] reportedAt, int n) + { + lock (_gate) + { + return Enumerable.Range(0, reportedAt.Length) + .Where(i => _folderCount[i] >= n) + .Select(i => Stopwatch.GetElapsedTime(reportedAt[i], _folderLast[i]).TotalMilliseconds) + .ToArray(); + } + } + + public double[] ServerLatenciesMs(long[] reportedAt) + { + lock (_gate) + { + return Enumerable.Range(0, reportedAt.Length) + .Where(i => _serverFirst[i] != 0) + .Select(i => Stopwatch.GetElapsedTime(reportedAt[i], _serverFirst[i]).TotalMilliseconds) + .ToArray(); + } + } + } +}